From d7eb8c395f06b696417de18f0fec010f54ea8210 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 08:04:34 +0000 Subject: [PATCH 01/17] Bump actions/setup-node from 6 to 7 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 6 to 7. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 96bb0965..e6b3abc0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: fetch-depth: 0 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 22 cache: npm @@ -106,7 +106,7 @@ jobs: fetch-depth: 0 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 22 cache: npm From 32e7e9f201224a22e875878281c14b0a685101fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 08:05:35 +0000 Subject: [PATCH 02/17] chore(deps): bump actions/upload-artifact from 4 to 7 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4620c0c8..fe75dc61 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ jobs: # actually contain their entry points — a gem missing one installs and # resolves, then dies on require. - run: bundle exec rake build_all - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: gems path: pkg/*.gem From a073de339a4c16e3426a17af6a272885859aa57a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 08:05:47 +0000 Subject: [PATCH 03/17] chore(deps): bump softprops/action-gh-release from 2 to 3 Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 2 to 3. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/v2...v3) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4620c0c8..7ab75506 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: path: pkg/*.gem - name: Attach gems to the release if: startsWith(github.ref, 'refs/tags/') - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: pkg/*.gem generate_release_notes: true From 70ef33b2462c914d386ce1d6c93020e7a2872e1e Mon Sep 17 00:00:00 2001 From: unknown <2314530442@qq.com> Date: Fri, 28 Aug 2026 01:12:59 +0800 Subject: [PATCH 04/17] fix: reject unsupported MCP transport methods --- actionagent/config/routes.rb | 3 +++ actionagent/test/engine_integration_test.rb | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/actionagent/config/routes.rb b/actionagent/config/routes.rb index 4c42e059..7203d4c5 100644 --- a/actionagent/config/routes.rb +++ b/actionagent/config/routes.rb @@ -135,6 +135,9 @@ # dashboard API key rather than a session, so it sits outside the api # namespace's session-authenticated controllers. post "mcp", to: "api/mcp#create" + match "mcp", to: ->(_env) { [ 405, { "Allow" => "POST" }, [] ] }, + via: [ :get, :delete ], + constraints: ->(request) { request.delete? || !request.format.html? } # Everything else under the mount is a client-side route: render the # dashboard and let the browser resolve it. Anchored last so it can only diff --git a/actionagent/test/engine_integration_test.rb b/actionagent/test/engine_integration_test.rb index 054053ce..b324e9fc 100644 --- a/actionagent/test/engine_integration_test.rb +++ b/actionagent/test/engine_integration_test.rb @@ -115,6 +115,25 @@ def setup assert_includes response.body, "active-agent-dashboard" end + test "mcp endpoint rejects unsupported transport methods" do + get "/activeagents/mcp", headers: { "Accept" => "text/event-stream" } + + assert_response :method_not_allowed + assert_equal "POST", response.headers["Allow"] + + delete "/activeagents/mcp" + + assert_response :method_not_allowed + assert_equal "POST", response.headers["Allow"] + end + + test "mcp dashboard route still renders for browsers" do + get "/activeagents/mcp", headers: { "Accept" => "text/html" } + + assert_response :success + assert_includes response.body, "active-agent-dashboard" + end + test "dashboard refuses unauthenticated access in production when no auth is configured" do Rails.env.stub(:local?, false) do get "/activeagents/console/traces" From e46d4b2d02196240822e5cd041d8d6a15fc0d6c9 Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Mon, 14 Sep 2026 08:21:45 -0700 Subject: [PATCH 05/17] chore(release): activeagent and actionagent 1.6.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releases both gems from one tag, cut from main at 51c7feb7. A minor rather than the 1.5.3 originally planned: the cycle adds ActiveAgent::Base#current_user and the as(...) caller seam, SchemaTools .define/.undefine with a per-model registry, the active_agent:schema_tools generator, schema tools served over the MCP facade, caller inheritance through delegate_to, and the ungrounded_answer evaluation fault. Seven new public APIs across both gems is more surface than a patch number advertises, even though 1.5.2 shipped a feature as a patch. The engine's floor on the framework (activeagent >= 1.4) is unchanged and still correct. activeagents-telemetry stays at ~> 0.1: nothing this cycle calls a 0.3-only API, and ~> 0.1 already resolves the published 0.3.0. Verified: CI green on the base commit across Ruby 3.2/3.3/3.4 and rails7/rails8/railsmain (run 34736861631, 1978 runs, 0 failures, 0 errors); bin/lint clean at 542 files; rake build_all produces activeagent-1.6.0.gem (220 KB, 194 files) and actionagent-1.6.0.gem (512 KB, 111 files), both archives carrying VERSION = "1.6.0" and their asserted entry points. A local suite run reports 34 errors that CI does not. All are environmental, and docs/work/release-1.6.0/validation.md records both causes: placeholder keys in .env.test, and gemfiles/rails8.gemfile.lock being untracked, so CI resolves ruby_llm fresh while a local checkout keeps 1.16.0 — whose Message dropped the tool_calls=/input_tokens= writers that the provider test's StubProvider assigns. Worth its own branch; not a release blocker. Not published. Tag-driven release has never succeeded in this repository: every release.yml run fails at "No trusted publisher configured for this workflow found on rubygems.org", which is why 1.5.1 and 1.5.2 have no tags yet are on RubyGems. The archives are built for the owner to push by hand. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CKFMuPp7hiQMutW9N8nKST --- CHANGELOG.md | 27 ++++++++ actionagent/lib/action_agent/version.rb | 2 +- docs/work/release-1.6.0/branch.md | 55 +++++++++++++++++ docs/work/release-1.6.0/validation.md | 61 +++++++++++++++++++ .../schema-tools-follow-ups/milestones.md | 2 +- lib/active_agent/version.rb | 2 +- 6 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 docs/work/release-1.6.0/branch.md create mode 100644 docs/work/release-1.6.0/validation.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d78b69fc..fffa5c9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.6.0] - 2026-09-14 + +Releases `activeagent` and `actionagent` 1.6.0 from one tag. + +A minor, not a patch. The cycle that began after 1.5.2 gives an agent a +caller — `current_user`, carried from whatever authenticated the call into +every `before_action`, every tool, every delegated sub-agent, every run over +MCP and every evaluation replay — so an authorization gem has something to +decide against. Around it: schema tools defined at runtime rather than only +in a file, a generator that writes the first one, those tools served +directly over MCP, and an evaluation that calls a fabricated answer a fault +instead of grading it as an honest gap. That is new public surface in both +gems, which is a minor under semver even though 1.5.2 shipped a feature as a +patch. + +Two notes for upgrades. `tools_succeeded` is now awarded only for a tool the +scenario expected, so a suite that was quietly scoring wrong-tool runs as +partial successes will report lower — read the first run as a corrected +baseline. And `actor:` is now stripped from tool arguments and from +`params[params][actor]`: the caller is a property of the run, set once by +whatever authenticated it, and can no longer be named by the model or by a +client. + +The engine's floor on the framework (`activeagent >= 1.4`) is unchanged and +still correct: 1.6.0 satisfies it. + ### Added - **An evaluation replay runs as the evaluation's owner.** The scenario runner @@ -92,6 +118,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **The judge reads more of a scenario's notes** — 1,500 characters rather than 300 — because a suite's notes are often its rubric and the "must not" clause tends to come last. (#433) + ### Fixed - **The caller can no longer be named by the model, or by the client.** diff --git a/actionagent/lib/action_agent/version.rb b/actionagent/lib/action_agent/version.rb index 6a4b1871..cdc2ec61 100644 --- a/actionagent/lib/action_agent/version.rb +++ b/actionagent/lib/action_agent/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActionAgent - VERSION = "1.5.2" + VERSION = "1.6.0" end diff --git a/docs/work/release-1.6.0/branch.md b/docs/work/release-1.6.0/branch.md new file mode 100644 index 00000000..7397f937 --- /dev/null +++ b/docs/work/release-1.6.0/branch.md @@ -0,0 +1,55 @@ +# Release 1.6.0 branch + +- Branch: `release/1.6.0` +- Base: main commit `51c7feb7` +- Tag: `v1.6.0` (not yet pushed) +- Ships: `activeagent` 1.6.0 and `actionagent` 1.6.0 from one tag + +## Commits + +| Commit | Purpose | +|--------|---------| +| _(this branch)_ | Version bumps, the dated 1.6.0 changelog heading, and these release notes | + +## Version jump + +`activeagent` and `actionagent` both go 1.5.2 -> 1.6.0. A minor, not a +patch: the cycle adds `ActiveAgent::Base#current_user` and the `as(...)` +caller seam, `SchemaTools.define`/`undefine` with a per-model registry, the +`active_agent:schema_tools` generator, schema tools served over the MCP +facade, caller inheritance through `delegate_to`, and the +`ungrounded_answer` evaluation fault. That is new public surface in both +gems. + +1.5.2 shipped the whole SchemaTools feature as a patch, so the repository +has precedent either way; 1.6.0 is the owner's call for this cycle, made +because seven new APIs is more surface than a patch number advertises. + +The engine's floor on the framework (`activeagent >= 1.4`) is unchanged and +still correct: 1.6.0 satisfies it. `activeagents-telemetry` stays at +`~> 0.1`; nothing in this cycle calls a 0.3-only API, and `~> 0.1` already +resolves the published 0.3.0. + +## Behaviour changes worth calling out + +- `tools_succeeded` is awarded only for a tool the scenario expected. Suites + that were scoring wrong-tool runs as partial successes will report lower. +- `actor:` is stripped from tool arguments and from `params[params][actor]`, + so the caller cannot be named by the model or by a client. + +## Artifacts + +Built to `pkg/` with `bundle exec rake build_all`, **not pushed**: + +- `pkg/activeagent-1.6.0.gem` (220 KB, 194 files) +- `pkg/actionagent-1.6.0.gem` (512 KB, 111 files) + +Publish `activeagent` first; `actionagent` depends on it. + +## Publishing + +Tag-driven publishing does not currently work in this repository — see +`validation.md`. Every `release.yml` run to date has failed at +`No trusted publisher configured for this workflow found on rubygems.org`, +which is why 1.5.1 and 1.5.2 have no tags yet are on RubyGems. The owner +publishes these two archives by hand. diff --git a/docs/work/release-1.6.0/validation.md b/docs/work/release-1.6.0/validation.md new file mode 100644 index 00000000..9aabe6e6 --- /dev/null +++ b/docs/work/release-1.6.0/validation.md @@ -0,0 +1,61 @@ +# Release 1.6.0 validation + +## Verification gate: CI on the base commit + +The release is cut from `main` at `51c7feb7` (the #451 merge). CI is green on +that exact commit — run `34736861631`: + +| Job | Result | +|---|---| +| `test (3.2, gemfiles/rails7.gemfile)` | 1978 runs, 6525 assertions, 0 failures, 0 errors, 22 skips | +| `test (3.3, gemfiles/rails8.gemfile)` | 1978 runs, 6525 assertions, 0 failures, 0 errors, 22 skips | +| `test (3.4, gemfiles/rails8.gemfile)` | 1978 runs, 6525 assertions, 0 failures, 0 errors, 22 skips | +| `test (3.4, gemfiles/railsmain.gemfile)` | 1978 runs, 6525 assertions, 0 failures, 0 errors, 22 skips | +| `Test API Gems (3.4)` | 8 runs, 18 assertions, 0 failures, 0 errors | +| `lint` | success | + +`bin/lint` locally: 542 files inspected, no offenses. + +`bundle exec rake build_all` succeeds and both archives pass the task's own +content assertions (`lib/active_agent.rb`; `lib/action_agent.rb`, +`config/routes.rb`, `app/assets/builds/action_agent.{js,css}`). + +## Local suite: 34 errors, all environmental + +A local run on Ruby 3.4.9 reports `1982 runs, 0 failures, 34 errors, 21 +skips` — four more runs and 34 more errors than CI on the same commit. None +is release content. Two causes: + +1. **Placeholder API keys.** `.env.test` holds placeholders (its own comment + says CI holds the real keys), so tests that reach a live provider fail + with `Incorrect API key provided: test-ope***-key`. This accounts for the + `integration_test.rb` errors and several in `ruby_llm_provider_test.rb`. + +2. **`gemfiles/rails8.gemfile.lock` is not committed.** `git ls-files + gemfiles/` lists the `.gemfile` files only. CI therefore resolves + `ruby_llm` fresh at install time, while a local checkout keeps whatever + its untracked lock pinned — here 1.16.0, whose `RubyLLM::Message` exposes + `tool_calls` and `input_tokens` as readers but no longer as writers. + `test/providers/ruby_llm/ruby_llm_provider_test.rb`'s `StubProvider` + assigns them (`msg.input_tokens = 10`, `msg.tool_calls = {...}`), so it + raises `NoMethodError` locally and passes in CI. Same commit, same + declared dependency, different resolution. + +Cause 2 is worth fixing on its own branch — either commit the lock or adapt +the stub to construct a `Message` with those values rather than assigning +them. It is a CI/local divergence that hides real breakage, but it is not a +1.6.0 blocker and no code in this release touches it. + +## Local toolchain note + +`.tool-versions` says `ruby latest`, which resolves to 4.0.6 here. Rails 8.1 +cannot boot on it — `bin/test` dies at `cannot load such file -- +active_storage/engine` before running anything. CI tests 3.2/3.3/3.4 only. +Verify locally with `mise exec ruby@3.4.9`. + +## Not verified + +No host application was installed against the built 1.6.0 archives (the +1.5.2 release did do this). The gems build, contain their entry points, and +CI is green on the base commit; an end-to-end install check is still worth +doing before pushing if the owner wants the same bar as last cycle. diff --git a/docs/work/schema-tools-follow-ups/milestones.md b/docs/work/schema-tools-follow-ups/milestones.md index dc8db565..d29bd5bf 100644 --- a/docs/work/schema-tools-follow-ups/milestones.md +++ b/docs/work/schema-tools-follow-ups/milestones.md @@ -10,4 +10,4 @@ | M6 | Runtime definitions with a registry; descendants leak closed (#447, #441) | ⏳ CI after the main merge | | M7 | Delegated runs inherit the caller (#448) | ⏳ stacked on #443 | | M8 | Schema tools over MCP (#449, #439) | ⏳ stacked on #443 | -| M9 | A 1.5.3 release carrying all of it | not started — the release is the owner's call | +| M9 | A release carrying all of it | ⏳ prepared as 1.6.0 (`release/1.6.0`), awaiting the owner's `gem push` | diff --git a/lib/active_agent/version.rb b/lib/active_agent/version.rb index e43f3d46..f25d2c5d 100644 --- a/lib/active_agent/version.rb +++ b/lib/active_agent/version.rb @@ -1,3 +1,3 @@ module ActiveAgent - VERSION = "1.5.2" + VERSION = "1.6.0" end From ba782eb7d7cb6dadfb355233b0451a278fb507da Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 06:54:29 -0700 Subject: [PATCH 06/17] fix(tools,telemetry): range filters that compare, and instrumentation that installs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects with the same shape: each turns a broken run into a plausible-looking success rather than an error, so neither showed up as a test failure. SchemaTools#permitted_filters! validated a filter's column against the allowlist but passed its value through untouched. A range hash therefore reached `where` unrecognized, and Rails compiled `where(due_date: {"before" => x})` to `due_date = NULL` — a predicate no row satisfies. The tool answered `{count: 0}` with no error and the model reported that as fact: "0 overdue tickets" against a database holding four. Equality filters were unaffected, which is why it survived. Comparisons now build through Arel with the column's own type cast (before, after, lt, lte, gt, gte, on_or_before, on_or_after; two bounds together express a window), and an operator outside that roster raises UnpermittedAttribute rather than returning zero — the same "reject rather than silently ignore" rule an undeclared column already gets. Only date, datetime, time and numeric columns are offered a range: a lexical `>` on a name column answers a question nobody asked. The range form was also undiscoverable. filter_properties described a date column as a bare {type: "string", format: "date"}, so the surface could not express "before today" and a model asking correctly still had no way to ask. Comparable columns now advertise anyOf: [scalar, range object]. Separately, the railtie prepended GenerationInstrumentation only when Telemetry.enabled? was already true as railties ran — before config/initializers/*.rb. An app enabling telemetry from its own initializer, which is what the docs show, was never instrumented: enabled? answered true, local_storage was on, the trace model resolved, the store lambda wrote a row when called by hand, and no generation ever produced a span to store. Telemetry.configure now installs too when the resulting config is enabled; instrument_telemetry! is idempotent, so the two paths cannot double-prepend and initializer order stops mattering. Both fixes carry tests that fail without them: stashing the source changes takes schema_tools_test from 0 failures to 6 failures and 1 error, and instrumentation_install_test from 0 to 2 failures. Co-Authored-By: Claude Opus 5 (1M context) --- lib/active_agent/schema_tools.rb | 114 +++++++++++++++++- lib/active_agent/telemetry.rb | 22 ++++ test/schema_tools_test.rb | 98 +++++++++++++++ .../telemetry/instrumentation_install_test.rb | 59 +++++++++ 4 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 test/telemetry/instrumentation_install_test.rb diff --git a/lib/active_agent/schema_tools.rb b/lib/active_agent/schema_tools.rb index 66c8978c..55bd6531 100644 --- a/lib/active_agent/schema_tools.rb +++ b/lib/active_agent/schema_tools.rb @@ -68,6 +68,21 @@ class SchemaTools # ask for 10_000; this is what stops that from becoming the prompt. MAX_LIMIT = 100 + # Column types a range comparison is offered for. Strings and booleans + # are deliberately absent: a lexical `>` on a name column answers a + # question nobody asked. + RANGE_FILTERABLE_TYPES = %i[date datetime time integer float decimal].freeze + + # The comparison operators a range filter may use, mapped to the Arel + # predicate that builds them. Names are the ones models reach for + # unprompted (`before`/`after` for dates, `lt`/`gte` for numbers), so a + # reasonable guess resolves instead of erroring. + RANGE_OPERATORS = { + "before" => :lt, "after" => :gt, + "lt" => :lt, "lte" => :lteq, "gt" => :gt, "gte" => :gteq, + "on_or_before" => :lteq, "on_or_after" => :gteq + }.freeze + # Raised when a tool call names a column outside the declared allowlists, # or is otherwise outside the declared boundary. class UnpermittedAttribute < ArgumentError; end @@ -300,8 +315,13 @@ def relation_for(actor) # Validates and normalizes a filter hash against the allowlist. # + # A filter value is normally matched for equality. A Hash value instead + # declares a range — `{ "before" => "2026-01-01" }`, `{ "gte" => 10 }` — + # and may carry two bounds at once to express a window. + # # @api private - # @raise [UnpermittedAttribute] if any key is not declared filterable + # @raise [UnpermittedAttribute] if any key is not declared filterable, + # or a range names an operator that does not exist def permitted_filters!(arguments) filters = arguments.each_with_object({}) do |(key, value), memo| next if value.nil? @@ -318,6 +338,54 @@ def permitted_filters!(arguments) filters end + # Splits filters into equality pairs and range predicates. + # + # Kept separate from {.permitted_filters!} because the two halves are + # applied differently: equality goes to `where(hash)`, ranges have to be + # built through Arel. + # + # @api private + # @return [Array(Hash, Array)] + def partition_filters!(filters) + equality = {} + ranges = [] + + filters.each do |column, value| + if value.is_a?(Hash) + ranges.concat(range_predicates!(column, value)) + else + equality[column] = value + end + end + + [ equality, ranges ] + end + + # Builds Arel predicates for one column's range hash. + # + # Rails silently turns `where(col: { "before" => x })` into `col = NULL`, + # which matches nothing and reports zero rather than failing — the worst + # outcome for an agent, which reads it as a truthful empty answer. So an + # unknown operator is rejected loudly here instead. + # + # @api private + # @raise [UnpermittedAttribute] on an unknown operator + def range_predicates!(column, value) + arel = @model.arel_table[column] + type = @model.type_for_attribute(column) + + value.map do |operator, operand| + predicate = RANGE_OPERATORS[operator.to_s] + unless predicate + raise UnpermittedAttribute, + "`#{operator}` is not a valid comparison for `#{column}`. " \ + "Allowed comparisons: #{RANGE_OPERATORS.keys.join(", ")}" + end + + arel.public_send(predicate, type.cast(operand)) + end + end + # Projects a record down to the declared return columns. # # The projection happens in SQL (+select+) as well as here, but the Ruby @@ -386,7 +454,40 @@ def filter_properties ) properties = schema[:schema][:properties] - filterable.index_with { |column| (properties[column] || { type: "string" }).deep_dup } + filterable.index_with do |column| + scalar = (properties[column] || { type: "string" }).deep_dup + range_filterable?(column) ? with_range_form(column, scalar) : scalar + end + end + + # Dates, times and numbers are the columns a question like "overdue" or + # "more than 10" actually needs a comparison on. + def range_filterable?(column) + RANGE_FILTERABLE_TYPES.include?(@model.type_for_attribute(column).type) + end + + # Offers a column as either a scalar (equality) or a range object. + # + # Without this the range form works but is undiscoverable: a model shown + # only `{type: "string", format: "date"}` has no way to know it may ask + # for `before`, and answers date questions with an equality match or no + # filter at all. + def with_range_form(column, scalar) + operand = scalar.slice(:type, :format) + description = scalar[:description] + + { + description: [ description, "Accepts an exact value, or a range object such as " \ + "{\"before\": ...} / {\"gte\": ...} (#{RANGE_OPERATORS.keys.join(", ")})." ].compact.join(" "), + anyOf: [ + scalar.except(:description), + { + type: "object", + properties: RANGE_OPERATORS.keys.index_with { operand.dup }, + additionalProperties: false + } + ] + } end def resource_name @@ -415,10 +516,10 @@ def define_find_tool ) define_singleton_method(name) do |actor: nil, limit: nil, **arguments| - filters = permitted_filters!(arguments) + equality, ranges = partition_filters!(permitted_filters!(arguments)) capped = normalize_limit(limit) - relation = relation_for(actor).where(filters) + relation = ranges.reduce(relation_for(actor).where(equality)) { |rel, p| rel.where(p) } # One extra row distinguishes "exactly at the limit" from "more than # the limit", without a second COUNT query. records = relation.limit(capped + 1).to_a @@ -443,9 +544,10 @@ def define_count_tool ) define_singleton_method(name) do |actor: nil, **arguments| - filters = permitted_filters!(arguments) + equality, ranges = partition_filters!(permitted_filters!(arguments)) + relation = ranges.reduce(relation_for(actor).where(equality)) { |rel, p| rel.where(p) } - { count: relation_for(actor).where(filters).count } + { count: relation.count } end end diff --git a/lib/active_agent/telemetry.rb b/lib/active_agent/telemetry.rb index 0a4d96ee..e6731143 100644 --- a/lib/active_agent/telemetry.rb +++ b/lib/active_agent/telemetry.rb @@ -78,9 +78,31 @@ def configuration # end def configure yield configuration if block_given? + install_instrumentation! if enabled? configuration end + # Installs generation instrumentation on ActiveAgent::Base. + # + # The railtie also does this at boot, but only for configuration already + # loaded by then (activeagent.yml, config.active_agent.telemetry). A host + # app that enables telemetry from its own initializer runs *after* + # railties, so that check has already seen `enabled? == false` and + # skipped the install — leaving telemetry enabled but nothing + # instrumented, and so no traces despite a valid local_store. Calling it + # from {.configure} as well makes the install order-independent; + # `instrument_telemetry!` is idempotent, so the two paths cannot + # double-prepend. + # + # @api private + # @return [void] + def install_instrumentation! + return unless defined?(ActiveAgent::Base) + + ActiveAgent::Base.include(Instrumentation) + ActiveAgent::Base.instrument_telemetry! + end + # Resets the configuration to defaults. # # @return [Configuration] New default configuration diff --git a/test/schema_tools_test.rb b/test/schema_tools_test.rb index aec2704c..b03b5b5e 100644 --- a/test/schema_tools_test.rb +++ b/test/schema_tools_test.rb @@ -453,4 +453,102 @@ def resolve = @actor ? @scope.all : @scope.none Class.new(ActiveAgent::SchemaTools) { scope_by_policy } end end + + # --- Range filters ----------------------------------------------------- + # + # Rails turns `where(col: {"before" => x})` into `col = NULL`, so before + # these were supported a range filter matched nothing and reported zero. + # An agent reads that as a truthful empty answer, which is why the silent + # case is tested as carefully as the working one. + + class DatedPostTools < ActiveAgent::SchemaTools + model Post + filterable :published_at, :published + returns :id, :title, :published_at + end + + test "range filter compares instead of matching nothing" do + Post.delete_all + old = Post.create!(title: "Old", content: "body", user: @alice, published_at: 10.days.ago) + Post.create!(title: "New", content: "body", user: @alice, published_at: 1.day.from_now) + + result = DatedPostTools.call("find_posts", published_at: { "before" => Time.current.iso8601 }) + + assert_equal 1, result[:count] + assert_equal [ old.title ], result[:results].map { |r| r[:title] } + end + + test "count applies a range filter" do + Post.delete_all + Post.create!(title: "Old", content: "body", user: @alice, published_at: 10.days.ago) + Post.create!(title: "New", content: "body", user: @alice, published_at: 1.day.from_now) + + assert_equal 1, DatedPostTools.call("count_posts", published_at: { "after" => Time.current.iso8601 })[:count] + end + + test "range filter accepts two bounds as a window" do + Post.delete_all + Post.create!(title: "Way old", content: "body", user: @alice, published_at: 30.days.ago) + inside = Post.create!(title: "Inside", content: "body", user: @alice, published_at: 5.days.ago) + Post.create!(title: "Future", content: "body", user: @alice, published_at: 5.days.from_now) + + result = DatedPostTools.call( + "find_posts", + published_at: { "after" => 10.days.ago.iso8601, "before" => Time.current.iso8601 } + ) + + assert_equal 1, result[:count] + assert_equal [ inside.title ], result[:results].map { |r| r[:title] } + end + + test "range filter combines with an equality filter" do + Post.delete_all + Post.create!(title: "Old published", content: "body", user: @alice, published_at: 10.days.ago, published: true) + Post.create!(title: "Old draft", content: "body", user: @alice, published_at: 10.days.ago, published: false) + + result = DatedPostTools.call( + "find_posts", published: true, published_at: { "before" => Time.current.iso8601 } + ) + + assert_equal 1, result[:count] + assert_equal [ "Old published" ], result[:results].map { |r| r[:title] } + end + + test "rejects an unknown comparison rather than reporting zero" do + Post.delete_all + Post.create!(title: "Old", content: "body", user: @alice, published_at: 10.days.ago) + + result = DatedPostTools.call("find_posts", published_at: { "roughly_before" => Time.current.iso8601 }) + + assert_match(/not a valid comparison/, result[:error]) + refute result.key?(:results), "must not return records when a comparison is rejected" + end + + test "a rejected comparison never silently widens or narrows the answer" do + Post.delete_all + Post.create!(title: "Old", content: "body", user: @alice, published_at: 10.days.ago) + + # The failure this guards: returning {count: 0} (silently narrowed) or the + # unfiltered set (silently widened) instead of an error. + result = DatedPostTools.call("count_posts", published_at: { "bogus" => "2026-01-01" }) + + assert result.key?(:error) + refute result.key?(:count) + end + + test "range filters are offered on comparable columns only" do + properties = DatedPostTools.tool_definitions.find { |d| d[:name] == "find_posts" } + .dig(:parameters, :properties) + + assert properties[:published_at].key?(:anyOf), "a datetime column must offer the range form" + refute properties[:published].key?(:anyOf), "a boolean column must not offer a range form" + end + + test "the advertised range operators are the ones accepted" do + properties = DatedPostTools.tool_definitions.find { |d| d[:name] == "find_posts" } + .dig(:parameters, :properties) + advertised = properties[:published_at][:anyOf].last[:properties].keys.map(&:to_s) + + assert_equal ActiveAgent::SchemaTools::RANGE_OPERATORS.keys.sort, advertised.sort + end end diff --git a/test/telemetry/instrumentation_install_test.rb b/test/telemetry/instrumentation_install_test.rb new file mode 100644 index 00000000..d734feea --- /dev/null +++ b/test/telemetry/instrumentation_install_test.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require "test_helper" + +# The railtie installs generation instrumentation at boot, but it can only +# see configuration that is already loaded by then. A host app that enables +# telemetry from its own `config/initializers/*.rb` runs *after* railties, so +# that check had already seen `enabled? == false` and skipped the install — +# leaving telemetry enabled, a valid local_store, and no traces at all. +# +# Configure is the one funnel every such app goes through, so the install +# happens there too and initializer order stops mattering. +class InstrumentationInstallTest < ActiveSupport::TestCase + setup do + @original = ActiveAgent::Telemetry.configuration + ActiveAgent::Telemetry.reset_configuration! + end + + teardown do + ActiveAgent::Telemetry.instance_variable_set(:@configuration, @original) + end + + def instrumented? + ActiveAgent::Base.ancestors.any? { |m| m.name.to_s.include?("GenerationInstrumentation") } + end + + test "enabling telemetry after boot installs instrumentation" do + ActiveAgent::Telemetry.configure do |config| + config.enabled = true + config.local_storage = true + end + + assert ActiveAgent::Telemetry.enabled?, "precondition: telemetry is enabled" + assert instrumented?, "enabling telemetry must install generation instrumentation" + end + + test "repeated configure calls do not prepend instrumentation twice" do + 2.times do + ActiveAgent::Telemetry.configure do |config| + config.enabled = true + config.local_storage = true + end + end + + matches = ActiveAgent::Base.ancestors.count { |m| m.name.to_s.include?("GenerationInstrumentation") } + assert_equal 1, matches, "instrument_telemetry! must stay idempotent across configure calls" + end + + test "configuring without enabling does not instrument" do + # Guards the inverse mistake: installing unconditionally would trace for + # apps that deliberately left telemetry off. + ActiveAgent::Telemetry.reset_configuration! + before = instrumented? + + ActiveAgent::Telemetry.configure { |config| config.sample_rate = 0.5 } + + assert_equal before, instrumented?, "a disabled configure must not install instrumentation" + end +end From 877ca0f6dda182247c8738ab05b47eff4c180f9f Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 07:25:21 -0700 Subject: [PATCH 07/17] fix(dashboard): an observed agent is registered once, not once per trace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AgentRegistrar deduplicates within Agent.for_owner(owner), and a single-tenant dashboard resolves no owner per trace — deliberately, since it owns everything itself. But for_owner(nil) returns `none` whenever an owner association is configured, which is the ordinary `owned_by :user` install. So the dedupe lookup matched nothing and every ingest registered another copy of the same agent, each taking a random slug suffix from the collision-breaker in observed_slug. MAX_OBSERVED_PER_OWNER did not contain it either: the count that cap reads was taken through the same empty relation, so it was always zero. Invisible until now because it takes real telemetry to see — an eighteen-call evaluation left eighteen identical EvaluationJudgeAgent rows. The existing "reuses the agent record across repeated ingests" test passes because this dummy app configures no owner association at all, where for_owner(nil) is `all`; the bug needs a host that configures one and runs single-tenant. Registration now scopes to the whole table when there is no owner to resolve, which is what for_owner already does for a model with no owner association. Multi-tenant behaviour is untouched: the registrar still returns early when an owner cannot be resolved and multi_tenant is on. Co-Authored-By: Claude Opus 5 (1M context) --- .../services/action_agent/agent_registrar.rb | 16 ++++++++++++++- actionagent/test/telemetry_trace_test.rb | 20 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/actionagent/app/services/action_agent/agent_registrar.rb b/actionagent/app/services/action_agent/agent_registrar.rb index 16675f4c..4d4d39fe 100644 --- a/actionagent/app/services/action_agent/agent_registrar.rb +++ b/actionagent/app/services/action_agent/agent_registrar.rb @@ -77,8 +77,22 @@ def owner_for_trace ActionAgent.multi_tenant? ? @trace.try(:account) : nil end + # The set an observed agent is deduplicated within. + # + # `for_owner(nil)` is `none` — correct for a multi-tenant read, where an + # unresolved tenant must see nothing. But a single-tenant dashboard has no + # owner to resolve and legitimately registers with `owner` nil (see + # #owner_for_trace), and `none` makes every dedupe lookup miss: each trace + # created another copy of the same agent, and the MAX_OBSERVED_PER_OWNER + # cap never engaged because the count it reads was always zero. Scoping to + # the whole table when there is no owner concept is what `for_owner` + # already does for a model with no owner association. + def agents_for_owner(owner) + owner.nil? && !ActionAgent.multi_tenant? ? Agent.all : Agent.for_owner(owner) + end + def find_or_create_agent(owner) - agents = Agent.for_owner(owner) + agents = agents_for_owner(owner) existing = agents.find_by( service_name: @trace.service_name, agent_class_name: agent_class, diff --git a/actionagent/test/telemetry_trace_test.rb b/actionagent/test/telemetry_trace_test.rb index 28aaa3d5..78060ea3 100644 --- a/actionagent/test/telemetry_trace_test.rb +++ b/actionagent/test/telemetry_trace_test.rb @@ -216,6 +216,26 @@ def prompt_span(roster) assert_equal 1, ActionAgent::Agent.where(agent_class_name: "SupportAgent").count end + # The case above passes with no owner association configured, where + # `for_owner(nil)` is `all`. A single-tenant host that DOES configure one — + # the ordinary Rails app, `owned_by :user` with multi_tenant off — resolves + # no owner per trace, and `for_owner(nil)` is then `none`: the dedupe lookup + # missed every time and each ingest registered another copy of one agent, + # with MAX_OBSERVED_PER_OWNER never engaging because the count it reads was + # always zero. Asserted on the registrar's own scoping seam, because the + # symptom depends on a host's owner model rather than on this dummy app's. + test "the dedupe scope is the whole table when there is no owner to resolve" do + trace = ActionAgent::TelemetryTrace.create_from_payload(payload(spans: [ root_span ])) + registrar = ActionAgent::AgentRegistrar.new(trace) + + scope = registrar.send(:agents_for_owner, nil) + + refute_equal "SELECT 1 AS one WHERE 1=0", scope.limit(1).to_sql, + "a single-tenant registrar must not dedupe against `none`" + assert_equal ActionAgent::Agent.count, scope.count, + "an unowned single-tenant ingest deduplicates against every agent row" + end + # Exercises AgentRegistrar's own rescue rather than a stub of it: a broken # registration must never cost a host app its telemetry. test "registration failure does not fail ingest" do From cfda4056b32cce6e156287d440078e90f4be7240 Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 07:25:32 -0700 Subject: [PATCH 08/17] chore(release): activeagent and actionagent 1.6.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releases both gems from one tag, cut from release/1.6.0 at 877ca0f6. A patch, not a minor: no new public surface and no behaviour change for anything that already worked. The cycle is three defects that share a failure mode — a date filter that matched nothing reported zero instead of raising, telemetry that was enabled but never instrumented wrote no traces while every configuration signal read healthy, and an observed agent was re-registered on every trace because the dedupe lookup ran through a relation that can never match. All three produce a confident wrong answer or a silently wrong row rather than an error, which is why none appeared in a suite. Two upgrade notes. A suite filtering on a date column will report different numbers after upgrading, because the filter now compares instead of matching nothing; read the first run as a corrected baseline. And duplicate observed agent rows already written are not cleaned up by the upgrade — delete them once. The engine's floor on the framework (activeagent >= 1.4) is unchanged and still correct; activeagents-telemetry stays at ~> 0.1. Verified: bin/lint clean at 543 files; bin/test 1994 runs, 6489 assertions, 2 failures and 34 errors, matching the pre-change baseline exactly (the failures are the RubyLLM suite, whose optional dependency is absent locally, plus one ordering-sensitive dashboard test that fails identically with and without these changes). Each fix carries a test that fails without it. The built archives were installed as real gems into the support-hub app and the full telemetry and evaluation path exercised end to end against live OpenRouter calls. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 51 +++++++++++++++++++++++++ actionagent/lib/action_agent/version.rb | 2 +- lib/active_agent/version.rb | 2 +- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fffa5c9e..097e6fd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,57 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.6.1] - 2026-09-16 + +Releases `activeagent` and `actionagent` 1.6.1 from one tag. + +A patch for two defects that share a failure mode: each one turns a broken +run into a plausible-looking success rather than an error. A date filter that +matched nothing reported zero instead of raising, and an agent reported that +zero as fact; telemetry that was enabled but never instrumented wrote no +traces while every configuration signal read healthy. Neither surfaced in a +test suite, because neither produces a failure — only a confident wrong +answer and an empty table. + +No new public surface and no behaviour change for anything that was already +working, so a patch under semver. Suites that filter on a date column will +report different — correct — numbers after upgrading; read the first run as a +corrected baseline. + +### Fixed + +- **A range filter on a `SchemaTools` column no longer matches nothing and + reports zero.** `permitted_filters!` validated the column against the + allowlist but passed the value through untouched, so a range hash reached + `where` unrecognized and Rails compiled `where(due_date: {"before" => x})` + to `due_date = NULL` — a predicate that matches no row. The tool returned + `{count: 0}` with no error and the model read it as a truthful empty + answer: "0 overdue tickets" against a database holding four. Equality + filters were unaffected, which is why this went unnoticed. Comparisons are + now built through Arel with the column's own type cast, under the operators + `before`, `after`, `lt`, `lte`, `gt`, `gte`, `on_or_before` and + `on_or_after`; two bounds may be given together to express a window; and an + operator outside that set raises `UnpermittedAttribute` rather than + returning zero, consistent with how an undeclared column is already + rejected. Ranges are offered for date, datetime, time and numeric columns + only — a lexical `>` on a name column answers a question nobody asked. +- **A range filter is now discoverable.** `filter_properties` described a date + column as a bare `{type: "string", format: "date"}`, so the tool surface + could not express "before today" at all and a model asking the question + correctly still had no way to ask it. Comparable columns are now offered as + `anyOf: [scalar, range object]`, with the operator roster in the schema. +- **Telemetry enabled from a host app's initializer now installs + instrumentation.** The railtie prepended `GenerationInstrumentation` only + when `Telemetry.enabled?` was already true as railties ran — before + `config/initializers/*.rb`. An app that configures telemetry in its own + initializer, which is what the documentation shows, was therefore never + instrumented: `enabled?` answered true, `local_storage` was on, the trace + model resolved and the store lambda worked when called directly, and no + generation ever produced a span to store. `configure` now installs as well + when the resulting configuration is enabled; `instrument_telemetry!` is + idempotent, so the railtie path and the configure path cannot + double-prepend and initializer order stops mattering. + ## [1.6.0] - 2026-09-14 Releases `activeagent` and `actionagent` 1.6.0 from one tag. diff --git a/actionagent/lib/action_agent/version.rb b/actionagent/lib/action_agent/version.rb index cdc2ec61..c6a3de41 100644 --- a/actionagent/lib/action_agent/version.rb +++ b/actionagent/lib/action_agent/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActionAgent - VERSION = "1.6.0" + VERSION = "1.6.1" end diff --git a/lib/active_agent/version.rb b/lib/active_agent/version.rb index f25d2c5d..1a99526f 100644 --- a/lib/active_agent/version.rb +++ b/lib/active_agent/version.rb @@ -1,3 +1,3 @@ module ActiveAgent - VERSION = "1.6.0" + VERSION = "1.6.1" end From 8a2364bc85d1b9922035ce6abb3f174567189872 Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 11:31:27 -0700 Subject: [PATCH 09/17] fix(dashboard): an evaluation created on MySQL can be run MySQL cannot give a JSON column a default, so an evaluation saved there without `config` reads it back as nil, and `compare_models` raised before the runner did anything else. `criteria` had the same gap on the paths that call `partition`, `select` or `reject` on it. `Evaluation#config` and `#criteria` now read as the empty value their column default supplies on other databases, which also covers rows that already exist. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 5 ++++ .../app/models/action_agent/evaluation.rb | 11 +++++++++ actionagent/test/evaluation_test.rb | 23 +++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 actionagent/test/evaluation_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index d78b69fc..fa449d51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 clause tends to come last. (#433) ### Fixed +- **An evaluation created on MySQL can be run.** MySQL cannot give a JSON + column a default, so an evaluation saved there without `config` read it + back as `nil`, and `compare_models` raised before the runner did anything + else. `config` and `criteria` now read as the empty value their column + default supplies on other databases. (#417) - **The caller can no longer be named by the model, or by the client.** `actor:` reached `AgentToolbox.call` in the same keyword namespace as the arguments a provider parsed out of a model's tool call, and diff --git a/actionagent/app/models/action_agent/evaluation.rb b/actionagent/app/models/action_agent/evaluation.rb index febcaf2f..8533dae0 100644 --- a/actionagent/app/models/action_agent/evaluation.rb +++ b/actionagent/app/models/action_agent/evaluation.rb @@ -35,6 +35,17 @@ class Evaluation < ApplicationRecord scope :recent, -> { order(updated_at: :desc) } + # MySQL cannot give a JSON column a default, so a row inserted there + # without `criteria` or `config` reads back nil. Both readers answer with + # the empty value the column default supplies on other databases. + def criteria + super || [] + end + + def config + super || {} + end + def latest_run evaluation_runs.order(created_at: :desc).first end diff --git a/actionagent/test/evaluation_test.rb b/actionagent/test/evaluation_test.rb new file mode 100644 index 00000000..303ec37a --- /dev/null +++ b/actionagent/test/evaluation_test.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "test_helper" + +# MySQL cannot give a JSON column a default, so an evaluation saved there +# without config or criteria reads them back as nil. The readers answer with +# the empty value the column default supplies on other databases. The test +# database honours that default, so the nil is assigned rather than persisted. +class ActionAgentEvaluationTest < ActiveSupport::TestCase + test "a nil config reads as empty and names no comparison models" do + evaluation = ActionAgent::Evaluation.new(config: nil) + + assert_equal({}, evaluation.config) + assert_equal [], evaluation.compare_models + end + + test "nil criteria read as empty" do + evaluation = ActionAgent::Evaluation.new(criteria: nil) + + assert_equal [], evaluation.criteria + assert_equal [], evaluation.llm_criteria + end +end From 53bd9b7b606002c4496a64a205de0a01d653a593 Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 11:31:27 -0700 Subject: [PATCH 10/17] refactor(dashboard): build the judge class inside its memoization `judge_class` resolved the provider, model and credentials on every call and then discarded them whenever `@judge_class` was already set. The work now happens once, inside the memoized block, and the test pins that the provider is resolved a single time per service instance. Co-Authored-By: Claude Fable 5.1 --- .../action_agent/evaluation_runner_service.rb | 20 ++++++++++--------- .../test/evaluation_runner_service_test.rb | 15 ++++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/actionagent/app/services/action_agent/evaluation_runner_service.rb b/actionagent/app/services/action_agent/evaluation_runner_service.rb index f26784a5..c3b03767 100644 --- a/actionagent/app/services/action_agent/evaluation_runner_service.rb +++ b/actionagent/app/services/action_agent/evaluation_runner_service.rb @@ -468,15 +468,17 @@ def owner end def judge_class - provider = judge_provider - model = @evaluation.judge_model.presence - options = {} - options[:model] = model if model - options.merge!(owner_provider_options(provider)) - - @judge_class ||= Class.new(ActiveAgent::Base) do - define_singleton_method(:name) { "EvaluationJudgeAgent" } - generate_with provider, **options + @judge_class ||= begin + provider = judge_provider + model = @evaluation.judge_model.presence + options = {} + options[:model] = model if model + options.merge!(owner_provider_options(provider)) + + Class.new(ActiveAgent::Base) do + define_singleton_method(:name) { "EvaluationJudgeAgent" } + generate_with provider, **options + end end end diff --git a/actionagent/test/evaluation_runner_service_test.rb b/actionagent/test/evaluation_runner_service_test.rb index 182b3d00..573434ef 100644 --- a/actionagent/test/evaluation_runner_service_test.rb +++ b/actionagent/test/evaluation_runner_service_test.rb @@ -20,6 +20,21 @@ def parse(content) end end + test "the judge class is built once per service instance" do + ActionAgent::Agent.delete_all + agent = ActionAgent::Agent.create!(name: "Judged", provider: "openai", model: "gpt-4o-mini") + evaluation = agent.evaluations.create!(name: "Quality", judge_kind: "llm", judge_model: "gpt-4o-mini", + criteria: [ { "key" => "quality", "type" => "llm_judge" } ]) + service = ActionAgent::EvaluationRunnerService.new(evaluation) + resolved = 0 + + service.stub(:judge_provider, -> { resolved += 1; :openai }) do + assert_same service.send(:judge_class), service.send(:judge_class) + end + + assert_equal 1, resolved + end + test "malformed or non-numeric judge scores are unscorable" do [ '{"score": "0.9"}', '{"score": true}', '{"score": null}', '{"score": {}}', '{"score": 0.9oops}', '{"score": 1e999}', '{"score": NaN}', "{}", "no json here", nil ].each do |content| From f2d64b4e4e645d162c9e5e475080a1a8a41e45d3 Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 11:31:27 -0700 Subject: [PATCH 11/17] docs(dashboard): document sign_in_path and sign_out_path Both options existed on `ActionAgent.configure` but neither the dashboard guide nor the generated initializer mentioned them, so a host that wanted a browser redirected to its own sign-in page had to read the engine's controller to find the setting. The Authentication section and the initializer template now show them beside `authentication_method`. Co-Authored-By: Claude Fable 5.1 --- .../action_agent/templates/action_agent.rb.erb | 8 ++++++++ docs/framework/dashboard.md | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/actionagent/lib/generators/action_agent/templates/action_agent.rb.erb b/actionagent/lib/generators/action_agent/templates/action_agent.rb.erb index b5b4d415..9e34a86d 100644 --- a/actionagent/lib/generators/action_agent/templates/action_agent.rb.erb +++ b/actionagent/lib/generators/action_agent/templates/action_agent.rb.erb @@ -31,6 +31,14 @@ ActionAgent.configure do |config| # config.authentication_method = nil # config.authentication_method = nil + # + # Where a browser is sent when it asks for a dashboard page without a valid + # session; unset, it gets a minimal session-expired page. API clients get a + # bare 401 either way. The sign-out path is what the header's "Sign out" + # item POSTs to (with _method=delete); unset, the item is not shown. + # + # config.sign_in_path = "/sign_in" + # config.sign_out_path = "/sign_out" # ========================================================================== # Ingest API authentication diff --git a/docs/framework/dashboard.md b/docs/framework/dashboard.md index 053e47df..90acb55b 100644 --- a/docs/framework/dashboard.md +++ b/docs/framework/dashboard.md @@ -510,6 +510,21 @@ ActionAgent.configure do |config| end ``` +A browser that asks for a dashboard page without a valid session is sent +to `config.sign_in_path` when one is set — your app's sign-in page — and +otherwise shown a minimal session-expired page. API and MCP clients get a +bare 401 either way. `config.sign_out_path` is the endpoint the header's +"Sign out" item POSTs to (with `_method=delete` and the CSRF token); the +engine has no session of its own, so leave it unset to hide the item. + +```ruby +ActionAgent.configure do |config| + config.authentication_method = ->(controller) { controller.authenticate_admin! } + config.sign_in_path = "/admin/sign_in" + config.sign_out_path = "/admin/sign_out" +end +``` + Or constrain the mount in `config/routes.rb`: ```ruby From a34258f92d2362a0ce1befe5c3f2c36ce1da63f7 Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 14:09:08 -0700 Subject: [PATCH 12/17] docs(changelog): file the merged entries under Unreleased Merged onto the release branch, the entries from the three pull requests landed inside the 1.6.0 section by surrounding context. They belong to the next release. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 91 +++++++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb8a877..9b57f54d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Agents have releases, and every trace, run and evaluation says which one + it ran under.** `ActiveAgent::Release` gives each agent class a digest of + what the model is given — provider and model, generation options minus + credentials, the actions, the prompt templates on disk, and the tools and + delegations it declares — so two deploys of the same agent share a digest + and any change to those inputs is a new one, with no number to bump. + `ActiveAgent::Release.revision` carries the deploy alongside (a git SHA; + read from `SERVICE_VERSION`, `GIT_SHA`, `KAMAL_VERSION` and friends when + not set). The instrumentation stamps `agent.version` and `agent.revision` + on every generation's root span, and every trace gets `service.version`. + In the dashboard, `rake action_agent:agents:release[REVISION]` cuts an + `AgentVersion` for each agent whose code changed since the last release — + idempotent, so it belongs in the deploy — `rake action_agent:agents:versions` + lists them, and `Agent#record_release!` is the call behind both for a host + that syncs agents its own way. Traces are pinned to the release their root + span names, runs and evaluation runs to the version current when they + started (`agent_version_id` on all three; the install generator emits the + migration). A version's JSON carries `release`, `release_digest` and + `revision`, so the Versions tab tells a deploy from an edit. For that to + reach a host's own agents, a trace from a class the host mirrors into the + dashboard is now attributed to that mirror — the registrar matched only on + service, class *and* action, so every code-path trace registered an + observed per-action twin beside the synced record and could never be + pinned to its release. + +### Fixed + +- **An evaluation created on MySQL can be run.** MySQL cannot give a JSON + column a default, so an evaluation saved there without `config` read it + back as `nil`, and `compare_models` raised before the runner did anything + else. `config` and `criteria` now read as the empty value their column + default supplies on other databases. (#417) +- **The Tools tab now says which schema tools an agent is offered, and + lets you change it.** The editor listed every schema tool as enabled and + read-only whatever `agent.tools` held — *"a checkbox that cannot add or + remove the tool is a control that changes nothing"* — while evaluations, + dashboard runs and the MCP facade offered exactly what that column named. + An agent whose roster had been emptied over the API ran a suite with no + tools (1/8, `expected tool not called ×6`) under a tab reading "12 + enabled". A schema tool's row now reads the roster and is switchable, and + every schema tool the host declares has a row, off unless the roster names + it — any agent may enable any of them, and a tool switched off has to keep + its row to be switched back on. A tool the agent class declares in code is + still reported rather than selected: the class offers it, and no checkbox + could change that. + ## [1.6.1] - 2026-09-16 Releases `activeagent` and `actionagent` 1.6.1 from one tag. @@ -86,30 +134,6 @@ still correct: 1.6.0 satisfies it. ### Added -- **Agents have releases, and every trace, run and evaluation says which one - it ran under.** `ActiveAgent::Release` gives each agent class a digest of - what the model is given — provider and model, generation options minus - credentials, the actions, the prompt templates on disk, and the tools and - delegations it declares — so two deploys of the same agent share a digest - and any change to those inputs is a new one, with no number to bump. - `ActiveAgent::Release.revision` carries the deploy alongside (a git SHA; - read from `SERVICE_VERSION`, `GIT_SHA`, `KAMAL_VERSION` and friends when - not set). The instrumentation stamps `agent.version` and `agent.revision` - on every generation's root span, and every trace gets `service.version`. - In the dashboard, `rake action_agent:agents:release[REVISION]` cuts an - `AgentVersion` for each agent whose code changed since the last release — - idempotent, so it belongs in the deploy — `rake action_agent:agents:versions` - lists them, and `Agent#record_release!` is the call behind both for a host - that syncs agents its own way. Traces are pinned to the release their root - span names, runs and evaluation runs to the version current when they - started (`agent_version_id` on all three; the install generator emits the - migration). A version's JSON carries `release`, `release_digest` and - `revision`, so the Versions tab tells a deploy from an edit. For that to - reach a host's own agents, a trace from a class the host mirrors into the - dashboard is now attributed to that mirror — the registrar matched only on - service, class *and* action, so every code-path trace registered an - observed per-action twin beside the synced record and could never be - pinned to its release. - **An evaluation replay runs as the evaluation's owner.** The scenario runner handed `Agent#test_execute` no caller, so every tool a replay called ran unattributed and a host scope answered empty — the suite graded an agent @@ -196,11 +220,6 @@ still correct: 1.6.0 satisfies it. ### Fixed -- **An evaluation created on MySQL can be run.** MySQL cannot give a JSON - column a default, so an evaluation saved there without `config` read it - back as `nil`, and `compare_models` raised before the runner did anything - else. `config` and `criteria` now read as the empty value their column - default supplies on other databases. (#417) - **The caller can no longer be named by the model, or by the client.** `actor:` reached `AgentToolbox.call` in the same keyword namespace as the arguments a provider parsed out of a model's tool call, and @@ -222,20 +241,6 @@ still correct: 1.6.0 satisfies it. hash naming both `provider` and `model` is now rebuilt as it was; a bare label is still parsed. The dashboard's "re-run" of a saved selection is the path this fixes. -- **The Tools tab now says which schema tools an agent is offered, and - lets you change it.** The editor listed every schema tool as enabled and - read-only whatever `agent.tools` held — *"a checkbox that cannot add or - remove the tool is a control that changes nothing"* — while evaluations, - dashboard runs and the MCP facade offered exactly what that column named. - An agent whose roster had been emptied over the API ran a suite with no - tools (1/8, `expected tool not called ×6`) under a tab reading "12 - enabled". A schema tool's row now reads the roster and is switchable, and - every schema tool the host declares has a row, off unless the roster names - it — any agent may enable any of them, and a tool switched off has to keep - its row to be switched back on. A tool the agent class declares in code is - still reported rather than selected: the class offers it, and no checkbox - could change that. - ## [1.5.2] - 2026-09-11 Releases `activeagent` and `actionagent` 1.5.2 from one tag. From f7290b00f7c359194af9c747ed993a57891e8e7f Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 14:10:02 -0700 Subject: [PATCH 13/17] chore(release): activeagent and actionagent 1.6.2 Releases both gems from one tag, cut from release/1.6.0. Agents gain releases (a digest of what the model is given, cut on deploy and pinned to every trace, run and evaluation run), the evaluation model reads its JSON columns as empty rather than nil on MySQL, the Tools tab reads the roster the runner reads, the MCP endpoint answers unsupported methods with 405, and sign_in_path / sign_out_path are documented. Three GitHub Actions bumps ride along. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 17 +++++++++++++++++ actionagent/lib/action_agent/version.rb | 2 +- lib/active_agent/version.rb | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b57f54d..a34df121 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.6.2] - 2026-09-16 + +Releases `activeagent` and `actionagent` 1.6.2 from one tag. + +Agents gain releases: a digest of everything the model is given, cut on +deploy and pinned to every trace, run and evaluation run, so a score is a +statement about a specific release and a regression is attributable to the +change that caused it. Around it, three dashboard fixes: an evaluation +created on MySQL can be run, the Tools tab reads the same `agent.tools` the +runner does, and the MCP endpoint answers an unsupported method with 405 +instead of the dashboard page. `sign_in_path` and `sign_out_path` are now +documented. + +Upgrading: the install generator emits a new `add_agent_releases` migration +(guarded column by column); run it. Cutting a release is +`rake action_agent:agents:release[REVISION]` in the deploy. + ### Added - **Agents have releases, and every trace, run and evaluation says which one diff --git a/actionagent/lib/action_agent/version.rb b/actionagent/lib/action_agent/version.rb index c6a3de41..f28a295a 100644 --- a/actionagent/lib/action_agent/version.rb +++ b/actionagent/lib/action_agent/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActionAgent - VERSION = "1.6.1" + VERSION = "1.6.2" end diff --git a/lib/active_agent/version.rb b/lib/active_agent/version.rb index 1a99526f..1965fc9a 100644 --- a/lib/active_agent/version.rb +++ b/lib/active_agent/version.rb @@ -1,3 +1,3 @@ module ActiveAgent - VERSION = "1.6.1" + VERSION = "1.6.2" end From 0dc93e5bb627465a1f1619783b5b4d87d26db9e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 21:44:39 +0000 Subject: [PATCH 14/17] Coerce container-valued query params and scrub handoff state on read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three hardenings the hosted platform made to its own copy of the dashboard controllers in September, ported to the engine so a host that mounts it gets the same behaviour. A query value can arrive as a container (`minutes[]=1&minutes[]=2`, or `page[x]=1`), and neither Array nor ActionController::Parameters responds to `to_i`. Every list the dashboard paginates or windows read those params directly and turned a malformed query into a 500. Api::BaseController gains integer_param and clamped_param: a multi-valued param means its first value, a nested object floors to the default. Applied to agent runs, agent analytics, the account analytics, interactions and session recordings. Sandboxes#compare read `providers` as a list. A bare string or a nested object raised NoMethodError; it is now a 400. A recording's show response returned the handoff state — a copy of the visitor's cookies and web storage — unscrubbed, both as its own key and nested inside metadata, while every other read path redacted. Both are now stripped of cookies, session_storage and local_storage; only #handoff returns them, to the owner, when they continue the session. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DkXcH6XVKyD5wYnpLuH9iE --- .../action_agent/api/agent_runs_controller.rb | 4 +- .../action_agent/api/agents_controller.rb | 8 +-- .../action_agent/api/analytics_controller.rb | 2 +- .../action_agent/api/base_controller.rb | 20 ++++++ .../api/interactions_controller.rb | 6 +- .../action_agent/api/sandboxes_controller.rb | 7 +- .../api/session_recordings_controller.rb | 31 +++++++-- actionagent/test/param_coercion_test.rb | 66 +++++++++++++++++++ .../test/session_recording_privacy_test.rb | 23 +++++++ 9 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 actionagent/test/param_coercion_test.rb diff --git a/actionagent/app/controllers/action_agent/api/agent_runs_controller.rb b/actionagent/app/controllers/action_agent/api/agent_runs_controller.rb index 805fb1d4..d30bc27e 100644 --- a/actionagent/app/controllers/action_agent/api/agent_runs_controller.rb +++ b/actionagent/app/controllers/action_agent/api/agent_runs_controller.rb @@ -33,8 +33,8 @@ def index scope = scope.where(agent_id: params[:agent_id]) if params[:agent_id].present? scope = scope.where(status: params[:status]) if params[:status].present? - page = (params[:page] || 1).to_i - per_page = (params[:per_page] || 20).to_i + page = integer_param(:page, default: 1) + per_page = integer_param(:per_page, default: 20) total = scope.count runs = scope.offset((page - 1) * per_page).limit(per_page) diff --git a/actionagent/app/controllers/action_agent/api/agents_controller.rb b/actionagent/app/controllers/action_agent/api/agents_controller.rb index 3c4a4bed..3a7d25dd 100644 --- a/actionagent/app/controllers/action_agent/api/agents_controller.rb +++ b/actionagent/app/controllers/action_agent/api/agents_controller.rb @@ -132,9 +132,9 @@ def restore # observed from telemetry have no AgentRun rows at all, so a runs-only # list showed them as empty while their scorecard reported real traffic. def runs - minutes = params[:minutes].presence&.then { |m| m.to_i.clamp(1, 60 * 24 * 90) } - page = (params[:page] || 1).to_i - per_page = (params[:per_page] || 20).to_i + minutes = integer_param(:minutes)&.clamp(1, 60 * 24 * 90) + page = integer_param(:page, default: 1) + per_page = integer_param(:per_page, default: 20) executions = AgentExecutions.new( agents: [ @agent ], @@ -282,7 +282,7 @@ def tool_roster # with all-zero metrics beside a card and a runs list reporting real # traffic. def analytics - days = (params[:days] || 30).to_i + days = integer_param(:days, default: 30) start_date = days.days.ago.beginning_of_day runs = @agent.agent_runs.where("created_at >= ?", start_date) diff --git a/actionagent/app/controllers/action_agent/api/analytics_controller.rb b/actionagent/app/controllers/action_agent/api/analytics_controller.rb index 1bf51b36..2f799cb5 100644 --- a/actionagent/app/controllers/action_agent/api/analytics_controller.rb +++ b/actionagent/app/controllers/action_agent/api/analytics_controller.rb @@ -5,7 +5,7 @@ module Api class AnalyticsController < BaseController # GET /api/analytics def index - days = (params[:days] || 30).to_i + days = integer_param(:days, default: 30) start_date = days.days.ago.beginning_of_day # Table names are interpolated rather than written literally: the diff --git a/actionagent/app/controllers/action_agent/api/base_controller.rb b/actionagent/app/controllers/action_agent/api/base_controller.rb index 960d6e7a..07ebd9df 100644 --- a/actionagent/app/controllers/action_agent/api/base_controller.rb +++ b/actionagent/app/controllers/action_agent/api/base_controller.rb @@ -129,6 +129,26 @@ def require_execution_enabled! render json: { error: "Agent execution is disabled on this dashboard" }, status: :forbidden end + # An integer query param. A value can arrive as a container + # (`minutes[]=1&minutes[]=2`, or `page[x]=1`), and neither Array nor + # ActionController::Parameters responds to `to_i`: reading them + # directly raised NoMethodError and turned a malformed query into a + # 500. A multi-valued param means its first value; anything else that + # is not a scalar falls back to the default. + def integer_param(name, default: nil) + raw = params[name] + raw = raw.first if raw.is_a?(Array) + return default if raw.blank? || !(raw.is_a?(String) || raw.is_a?(Numeric)) + + raw.to_s.to_i + end + + # integer_param, then clamped into [min, max]. Non-numeric input becomes + # 0 and is then clamped up to `min`. + def clamped_param(name, default:, min:, max:) + integer_param(name, default: default).clamp(min, max) + end + def not_found render json: { error: "Record not found" }, status: :not_found end diff --git a/actionagent/app/controllers/action_agent/api/interactions_controller.rb b/actionagent/app/controllers/action_agent/api/interactions_controller.rb index 17c1fec0..1bb7a479 100644 --- a/actionagent/app/controllers/action_agent/api/interactions_controller.rb +++ b/actionagent/app/controllers/action_agent/api/interactions_controller.rb @@ -16,7 +16,7 @@ class InteractionsController < BaseController # GET /api/interactions def index - limit = params.fetch(:limit, DEFAULT_LIMIT).to_i.clamp(1, 200) + limit = clamped_param(:limit, default: DEFAULT_LIMIT, min: 1, max: 200) contexts = interactions_scope .includes(:contextable) @@ -140,8 +140,8 @@ def traces_for_agent(agent_id) def window_minutes return @window_minutes if defined?(@window_minutes) - raw = params[:minutes].presence - @window_minutes = raw ? raw.to_i.clamp(1, MAX_WINDOW_MINUTES) : nil + raw = integer_param(:minutes) + @window_minutes = raw ? raw.clamp(1, MAX_WINDOW_MINUTES) : nil end def interactions_scope diff --git a/actionagent/app/controllers/action_agent/api/sandboxes_controller.rb b/actionagent/app/controllers/action_agent/api/sandboxes_controller.rb index 5600ac7f..42f4084d 100644 --- a/actionagent/app/controllers/action_agent/api/sandboxes_controller.rb +++ b/actionagent/app/controllers/action_agent/api/sandboxes_controller.rb @@ -18,11 +18,16 @@ class SandboxesController < BaseController # POST /api/sandboxes/compare # Run multiple providers in a single sandbox using parallel generation jobs def compare - providers = params[:providers] || %w[anthropic openai ollama] + providers = params[:providers].nil? ? %w[anthropic openai ollama] : params[:providers] task = params[:task] sandbox_id = params[:sandbox_id] return render json: { error: "Task required" }, status: :bad_request unless task.present? + # A bare string or a nested object is a malformed request, not a list + # of one provider: reading it as a list raised NoMethodError. + unless providers.is_a?(Array) && providers.all? { |name| name.is_a?(String) } + return render json: { error: "providers must be a list of provider names" }, status: :bad_request + end return render json: { error: "At least 2 providers required" }, status: :bad_request if providers.size < 2 # Validate providers diff --git a/actionagent/app/controllers/action_agent/api/session_recordings_controller.rb b/actionagent/app/controllers/action_agent/api/session_recordings_controller.rb index e54dcab7..c9fbf78b 100644 --- a/actionagent/app/controllers/action_agent/api/session_recordings_controller.rb +++ b/actionagent/app/controllers/action_agent/api/session_recordings_controller.rb @@ -13,6 +13,12 @@ class SessionRecordingsController < BaseController before_action :set_recording, only: [ :show, :actions, :snapshot, :export, :handoff ] + # Browser state that must never leave the server in a read response: + # the handoff state a recording carries is a copy of the visitor's + # cookies and web storage. Only #handoff returns it, to the owner, when + # they continue the session. + SENSITIVE_STATE_KEYS = %w[cookies session_storage local_storage].freeze + # GET /api/session_recordings # List recordings with optional filters def index @@ -36,8 +42,8 @@ def index end # Pagination - page = (params[:page] || 1).to_i - per_page = [ (params[:per_page] || 20).to_i, 100 ].min + page = integer_param(:page, default: 1) + per_page = [ integer_param(:per_page, default: 20), 100 ].min offset = (page - 1) * per_page total = recordings.count @@ -79,10 +85,10 @@ def actions # Support pagination for large recordings if params[:after_sequence].present? - actions = actions.where("sequence > ?", params[:after_sequence].to_i) + actions = actions.where("sequence > ?", integer_param(:after_sequence, default: 0)) end - limit = [ params[:limit]&.to_i || 100, 500 ].min + limit = [ integer_param(:limit, default: 100), 500 ].min actions = actions.limit(limit) render json: { @@ -332,7 +338,7 @@ def recording_detail(recording) created_at: recording.created_at.iso8601, updated_at: recording.updated_at.iso8601, timeline: recording.timeline, - handoff_state: recording.metadata["handoff_state"], + handoff_state: safe_handoff_state(recording.metadata["handoff_state"]), agent: recording.agent_run&.agent&.slice(:id, :name), sandbox_session: recording.sandbox_session&.summary } @@ -343,9 +349,20 @@ def first_screenshot_url(recording) action&.screenshot_url(expires_in: 1.hour) end + # Strips the browser state at the top level and inside handoff_state, + # which the model stores nested (a recording's metadata carries the + # handoff as one key), so a show response never ships a session cookie. def safe_metadata(metadata) - # Remove sensitive data from metadata - metadata.except("cookies", "session_storage", "local_storage") + safe = (metadata || {}).except(*SENSITIVE_STATE_KEYS) + return safe unless safe["handoff_state"].is_a?(Hash) + + safe.merge("handoff_state" => safe_handoff_state(safe["handoff_state"])) + end + + def safe_handoff_state(handoff_state) + return handoff_state unless handoff_state.is_a?(Hash) + + handoff_state.except(*SENSITIVE_STATE_KEYS) end def generate_visitor_id diff --git a/actionagent/test/param_coercion_test.rb b/actionagent/test/param_coercion_test.rb new file mode 100644 index 00000000..e139fdd5 --- /dev/null +++ b/actionagent/test/param_coercion_test.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require "test_helper" + +# A query value can arrive as a container (`minutes[]=1&minutes[]=2`, or +# `page[x]=1`), and neither Array nor ActionController::Parameters responds +# to `to_i`. Reading them directly raised NoMethodError and turned a +# malformed query into a 500 on every list the dashboard paginates or +# windows. A multi-valued param means its first value; a nested object is +# malformed and floors to the default. +class ParamCoercionTest < ActionDispatch::IntegrationTest + def setup + ActionAgent::Agent.delete_all + @agent = ActionAgent::Agent.create!(name: "Support", provider: "openai", model: "gpt-4o-mini") + @agent.agent_runs.create!(input_prompt: "hi", output: "hello", status: :complete) + end + + test "agent runs coerce container-valued minutes, page and per_page" do + get "/activeagents/api/agents/#{@agent.id}/runs", params: { minutes: [ 1, 2 ], page: { x: 1 }, per_page: [ 5 ] } + + assert_response :success, response.body + body = JSON.parse(response.body) + assert_equal 1, body["runs"].length + assert_equal 5, body["meta"]["per_page"] + assert_equal 1, body["meta"]["page"] + end + + test "agent analytics coerces a container-valued days param" do + get "/activeagents/api/agents/#{@agent.id}/analytics", params: { days: [ 7, 30 ] } + + assert_response :success, response.body + end + + test "interactions coerce container-valued minutes and limit" do + context = ActionAgent::AgentContext.create!(contextable: @agent, agent_name: "SupportAgent", action_name: "respond") + context.add_user_message("Where is order 88213?") + + get "/activeagents/api/interactions", params: { minutes: [ 60, 120 ], limit: { n: 10 } } + + assert_response :success, response.body + assert_equal 1, JSON.parse(response.body)["interactions"].length + end + + test "session recordings coerce container-valued page, per_page, after_sequence and limit" do + recording = ActionAgent::SessionRecording.start_user_session!(page_url: "https://example.com/") + recording.record_action!(action_type: "click", selector: "button") + + get "/activeagents/api/session_recordings", params: { page: [ 1 ], per_page: { n: 20 } } + assert_response :success, response.body + assert_equal 1, JSON.parse(response.body).dig("pagination", "page") + + get "/activeagents/api/session_recordings/#{recording.id}/actions", params: { after_sequence: [ 0 ], limit: { n: 5 } } + assert_response :success, response.body + assert_equal 1, JSON.parse(response.body)["actions"].size + end + + test "sandbox compare rejects a providers value that is not a list of names" do + sandbox = ActionAgent::SandboxSession.create!(session_id: SecureRandom.uuid, status: :ready, expires_at: 1.hour.from_now) + + post "/activeagents/api/sandboxes/compare", params: { task: "Take a screenshot", providers: "anthropic", sandbox_id: sandbox.session_id } + assert_response :bad_request, response.body + + post "/activeagents/api/sandboxes/compare", params: { task: "Take a screenshot", providers: { a: "anthropic" }, sandbox_id: sandbox.session_id } + assert_response :bad_request, response.body + end +end diff --git a/actionagent/test/session_recording_privacy_test.rb b/actionagent/test/session_recording_privacy_test.rb index cfa96127..2e953b94 100644 --- a/actionagent/test/session_recording_privacy_test.rb +++ b/actionagent/test/session_recording_privacy_test.rb @@ -48,6 +48,29 @@ def setup assert_nil exported["metadata"]["password"] assert_not_includes response.body, "hunter2secret" end + + # The handoff state is a copy of the visitor's browser: cookies and web + # storage. Stripping only the top level of the metadata left the same + # secrets readable one key down, and as the top-level handoff_state key. + test "show strips cookies and web storage from the nested handoff state" do + @recording.update!(metadata: @recording.metadata.merge( + "handoff_state" => { + "url" => "https://example.com/checkout", + "cookies" => [ { "name" => "_session", "value" => "sekrit-cookie" } ], + "local_storage" => { "auth_token" => "lst-secret" }, + "session_storage" => { "csrf" => "sst-secret" } + } + )) + + get "/activeagents/api/session_recordings/#{@recording.id}" + + assert_response :success + body = JSON.parse(response.body)["recording"] + assert_equal "https://example.com/checkout", body.dig("handoff_state", "url") + assert_nil body.dig("handoff_state", "cookies") + assert_nil body.dig("metadata", "handoff_state", "cookies") + %w[sekrit-cookie lst-secret sst-secret].each { |secret| assert_not_includes response.body, secret } + end end # In a per-user install the list has to show the recordings the caller can From 49a57cbf3c53f663bd020040ae1e50f3870aa1a7 Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Wed, 16 Sep 2026 14:57:45 -0700 Subject: [PATCH 15/17] test(telemetry): restore the global configuration after swapping the tracer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `swap_global_tracer` turned the global configuration's `enabled` and `local_storage` on and its callers' `ensure` called it again to "restore", so both tests left telemetry on with local storage for the rest of the process. With generation instrumentation on `ActiveAgent::Base`, every later generation in the suite was traced, delivered by the batching reporter's thread on its own connection, and committed — surviving the test transaction and surfacing as observed agents in whichever dashboard test ran next. The swap now returns the tracer and both flags, and `restore_global_tracer` puts all three back. Same-seed full suite on a clean database: 0 failures, no rows committed outside a transaction, none left on disk. Co-Authored-By: Claude Fable 5.1 --- .../test/telemetry_correlation_test.rb | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/actionagent/test/telemetry_correlation_test.rb b/actionagent/test/telemetry_correlation_test.rb index 6e0f2651..f2529b25 100644 --- a/actionagent/test/telemetry_correlation_test.rb +++ b/actionagent/test/telemetry_correlation_test.rb @@ -64,7 +64,7 @@ def setup end test "instrumented generations share the trace_id exposed in prompt_options" do - original_tracer = swap_global_tracer(ActiveAgent::Telemetry::Tracer.new(@configuration)) + previous = swap_global_tracer(ActiveAgent::Telemetry::Tracer.new(@configuration)) agent_class = Class.new(ApplicationAgent) do def self.name = "CorrelationProbeAgent" @@ -91,11 +91,11 @@ def ping assert_match(/\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/, trace.trace_id, "trace_id should be the prompt_options UUID, not a tracer-generated hex id") ensure - swap_global_tracer(original_tracer) + restore_global_tracer(previous) end test "instrumented generations record provider and model attributes" do - original_tracer = swap_global_tracer(ActiveAgent::Telemetry::Tracer.new(@configuration)) + previous = swap_global_tracer(ActiveAgent::Telemetry::Tracer.new(@configuration)) agent_class = Class.new(ApplicationAgent) do def self.name = "AttributeProbeAgent" @@ -122,7 +122,7 @@ def ping assert_equal "mock-model", llm.dig("attributes", "llm.model") assert_operator prompt_span.dig("attributes", "messages.count").to_i, :>=, 1 ensure - swap_global_tracer(original_tracer) + restore_global_tracer(previous) end private @@ -144,13 +144,28 @@ def stored_payload } end + # Points the framework's global tracer at +tracer+ with local storage on, + # returning the state to hand back to +restore_global_tracer+. Left as it + # is, `local_storage` keeps `enabled?` true for the rest of the process, + # and every later generation in the suite is traced and registered as an + # observed agent. def swap_global_tracer(tracer) - previous = ActiveAgent::Telemetry.instance_variable_get(:@tracer) + config = ActiveAgent::Telemetry.configuration + previous = { + tracer: ActiveAgent::Telemetry.instance_variable_get(:@tracer), + enabled: config.enabled, + local_storage: config.local_storage + } ActiveAgent::Telemetry.instance_variable_set(:@tracer, tracer) - previous_enabled = ActiveAgent::Telemetry.configuration.enabled - ActiveAgent::Telemetry.configuration.enabled = true - ActiveAgent::Telemetry.configuration.local_storage = true - @restore_enabled = previous_enabled + config.enabled = true + config.local_storage = true previous end + + def restore_global_tracer(previous) + config = ActiveAgent::Telemetry.configuration + ActiveAgent::Telemetry.instance_variable_set(:@tracer, previous[:tracer]) + config.enabled = previous[:enabled] + config.local_storage = previous[:local_storage] + end end From 8ebff5151fb0ac7b4ded06ff471ce7f778f5569d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 22:52:26 +0000 Subject: [PATCH 16/17] docs(changelog): file the dashboard API hardening under 1.6.2 The merge of #456 brings the container-valued query parameter coercion, the compare providers validation and the recording show scrub onto the release line; the 1.6.2 section now lists them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DkXcH6XVKyD5wYnpLuH9iE --- CHANGELOG.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a34df121..26dbb1e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,11 @@ Releases `activeagent` and `actionagent` 1.6.2 from one tag. Agents gain releases: a digest of everything the model is given, cut on deploy and pinned to every trace, run and evaluation run, so a score is a statement about a specific release and a regression is attributable to the -change that caused it. Around it, three dashboard fixes: an evaluation +change that caused it. Around it, five dashboard fixes: an evaluation created on MySQL can be run, the Tools tab reads the same `agent.tools` the -runner does, and the MCP endpoint answers an unsupported method with 405 +runner does, a container-valued query parameter is coerced instead of +raising, a recording's detail response no longer carries the visitor's +cookies and web storage, and the MCP endpoint answers an unsupported method with 405 instead of the dashboard page. `sign_in_path` and `sign_out_path` are now documented. @@ -71,6 +73,19 @@ Upgrading: the install generator emits a new `add_agent_releases` migration its row to be switched back on. A tool the agent class declares in code is still reported rather than selected: the class offers it, and no checkbox could change that. +- **A container-valued query parameter no longer 500s the dashboard API.** + `page`, `per_page`, `days`, `minutes`, `limit` and `after_sequence` were + read with `to_i`, which neither an Array (`minutes[]=1&minutes[]=2`) nor a + nested object (`page[x]=1`) answers. `Api::BaseController` now coerces + them: a multi-valued parameter means its first value, a nested object falls + back to the default, and the clamps that bounded the number still apply. + `sandboxes#compare` answers a `providers` value that is not a list of + names with a 400 instead of a `NoMethodError`. +- **A session recording's `show` no longer returns the visitor's cookies and + web storage.** Every other read path redacted the handoff state, but the + detail response carried `cookies`, `session_storage` and `local_storage` + unscrubbed, both as its own key and nested inside `metadata`. Both are now + stripped; only `#handoff` returns them, to the recording's owner. (#456) ## [1.6.1] - 2026-09-16 From f995cc9fb7c6ed5e696f88f835c03764ba5bf248 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 23:03:39 +0000 Subject: [PATCH 17/17] test: load Action Cable's config before any test can stub Rails.env Action Cable reads config/cable.yml for the current environment the first time its server class loads, and keeps what it read. The dummy app does not eager load in test, so that first time was whichever test first broadcast or rendered the dashboard. When the seed put engine_integration_test's staging case first, its stubbed Rails.env had no section in cable.yml: the config came back empty, the adapter fell back to redis, and every later broadcast in the process raised Gem::LoadError for redis-client (28 errors and failures on the rails-main job for 8ebff51, none on the same revision an hour earlier). The helper now loads the config under the real test environment before any test runs. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DkXcH6XVKyD5wYnpLuH9iE --- test/test_helper.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_helper.rb b/test/test_helper.rb index c32ba385..5745b6a8 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -17,6 +17,15 @@ require "webmock/minitest" require "minitest/mock" +# Action Cable reads config/cable.yml for the current environment the first +# time its server class loads, and keeps what it read. With eager loading off +# that first time is whichever test first broadcasts or renders the dashboard, +# and a test that stubs Rails.env to a name with no section in cable.yml +# (engine_integration_test's staging case) must not be it: the config would +# come back empty, the adapter would fall back to redis, and every later +# broadcast in the process would raise. Load it here, under the test env. +ActionCable.server.config.cable + # Extract full path and relative path from caller_info def extract_path_info(caller_info) if caller_info =~ /(.+):(\d+):in/