diff --git a/CHANGELOG.md b/CHANGELOG.md index d78b69fc..9ebd5e35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Agents have releases, and every trace, run and evaluation says which one + it ran under.** `ActiveAgent::Release` gives each agent class a digest of + what the model is given — provider and model, generation options minus + credentials, the actions, the prompt templates on disk, and the tools and + delegations it declares — so two deploys of the same agent share a digest + and any change to those inputs is a new one, with no number to bump. + `ActiveAgent::Release.revision` carries the deploy alongside (a git SHA; + read from `SERVICE_VERSION`, `GIT_SHA`, `KAMAL_VERSION` and friends when + not set). The instrumentation stamps `agent.version` and `agent.revision` + on every generation's root span, and every trace gets `service.version`. + In the dashboard, `rake action_agent:agents:release[REVISION]` cuts an + `AgentVersion` for each agent whose code changed since the last release — + idempotent, so it belongs in the deploy — `rake action_agent:agents:versions` + lists them, and `Agent#record_release!` is the call behind both for a host + that syncs agents its own way. Traces are pinned to the release their root + span names, runs and evaluation runs to the version current when they + started (`agent_version_id` on all three; the install generator emits the + migration). A version's JSON carries `release`, `release_digest` and + `revision`, so the Versions tab tells a deploy from an edit. For that to + reach a host's own agents, a trace from a class the host mirrors into the + dashboard is now attributed to that mirror — the registrar matched only on + service, class *and* action, so every code-path trace registered an + observed per-action twin beside the synced record and could never be + pinned to its release. - **An evaluation replay runs as the evaluation's owner.** The scenario runner handed `Agent#test_execute` no caller, so every tool a replay called ran unattributed and a host scope answered empty — the suite graded an agent diff --git a/actionagent/app/controllers/action_agent/api/agents_controller.rb b/actionagent/app/controllers/action_agent/api/agents_controller.rb index 3c4a4bed..0d10d9d6 100644 --- a/actionagent/app/controllers/action_agent/api/agents_controller.rb +++ b/actionagent/app/controllers/action_agent/api/agents_controller.rb @@ -532,7 +532,10 @@ def version_json(version, include_diff: false) change_summary: version.change_summary, created_by: version.created_by, created_at: version.created_at, - is_latest: version.latest? + is_latest: version.latest?, + release: version.release?, + release_digest: version.release_digest, + revision: version.revision } if include_diff && version.previous diff --git a/actionagent/app/models/action_agent/agent.rb b/actionagent/app/models/action_agent/agent.rb index bda784af..b049be63 100644 --- a/actionagent/app/models/action_agent/agent.rb +++ b/actionagent/app/models/action_agent/agent.rb @@ -219,6 +219,46 @@ def latest_version agent_versions.order(version_number: :desc).first end + # The most recent version cut from the agent's code, if any. + # @return [AgentVersion, nil] + def latest_release + agent_versions.releases.order(version_number: :desc).first + end + + # Cuts a version for a release of the agent's code, identified by the + # digest ActiveAgent::Release computes from what the model is given. + # Returns the existing version when the latest release already carries + # this digest — a redeploy of an unchanged agent is not a new version — + # so it is safe to call on every deploy. + # + # The version's snapshot is the dashboard configuration plus the release + # manifest under "release", so the Versions tab can diff two releases the + # same way it diffs two dashboard edits. + # + # @param digest [String] ActiveAgent::Release digest of the host class + # @param manifest [Hash, nil] the class's release manifest + # @param revision [String, nil] the deploy (git SHA, release label) + # @param released_by [String, nil] + # @return [AgentVersion] + def record_release!(digest:, manifest: nil, revision: nil, released_by: nil) + current = latest_release + if current && current.release_digest == digest + update_columns(release_digest: digest) if release_digest != digest + return current + end + + version = agent_versions.create!( + version_number: (latest_version&.version_number || 0) + 1, + change_summary: release_summary(digest, revision, current&.configuration_snapshot&.dig("release"), manifest), + configuration_snapshot: configuration_snapshot.merge("release" => manifest || {}), + release_digest: digest, + revision: revision, + created_by: released_by || "release" + ) + update_columns(release_digest: digest) + version + end + # Maps each historical instructions digest to the first version that # introduced it ("v3"), so run cohorts can label instruction changes with # real agent versions instead of raw hashes. @@ -396,6 +436,18 @@ def generate_slug end end + # "Release 1a2b3c4d5e6f · abc1234: templates, tools" — the digest, the + # deploy, and which parts of the manifest moved since the last release. + def release_summary(digest, revision, previous_manifest, manifest) + label = [ "Release #{digest}", revision.presence ].compact.join(" · ") + return "#{label}: first release" if previous_manifest.blank? || manifest.blank? + + changed = (previous_manifest.keys | manifest.stringify_keys.keys).select do |key| + previous_manifest[key] != manifest.stringify_keys[key] + end + changed.any? ? "#{label}: #{changed.sort.join(', ')}" : label + end + def create_initial_version agent_versions.create!( version_number: 1, diff --git a/actionagent/app/models/action_agent/agent_run.rb b/actionagent/app/models/action_agent/agent_run.rb index 7eeed015..cf6dc4a3 100644 --- a/actionagent/app/models/action_agent/agent_run.rb +++ b/actionagent/app/models/action_agent/agent_run.rb @@ -3,6 +3,10 @@ module ActionAgent class AgentRun < ApplicationRecord belongs_to :agent + # The version of the agent this run executed under — the latest at the + # time, since a run is against the agent as it is. + belongs_to :agent_version, optional: true + before_create { self.agent_version_id ||= agent&.latest_version&.id } # Raised when a caller hands a run files to attach in a host app that # has nowhere to keep them. diff --git a/actionagent/app/models/action_agent/agent_version.rb b/actionagent/app/models/action_agent/agent_version.rb index 8dd77e56..15db76c8 100644 --- a/actionagent/app/models/action_agent/agent_version.rb +++ b/actionagent/app/models/action_agent/agent_version.rb @@ -9,6 +9,9 @@ class AgentVersion < ApplicationRecord # Scopes scope :recent, -> { order(version_number: :desc) } + # Versions cut from the agent's code on deploy, as opposed to edits made + # in the dashboard. + scope :releases, -> { where.not(release_digest: [ nil, "" ]) } scope :by_version, ->(num) { where(version_number: num) } # Compare two versions @@ -36,6 +39,13 @@ def next_version end # Check if this is the latest version + # Whether this version was cut from the agent's code (it carries the + # release digest) rather than from a dashboard edit. + # @return [Boolean] + def release? + release_digest.present? + end + def latest? agent.latest_version&.id == id end diff --git a/actionagent/app/models/action_agent/evaluation_run.rb b/actionagent/app/models/action_agent/evaluation_run.rb index 4f02e315..f1e61710 100644 --- a/actionagent/app/models/action_agent/evaluation_run.rb +++ b/actionagent/app/models/action_agent/evaluation_run.rb @@ -8,6 +8,10 @@ module ActionAgent # See #average_score, which is what has to tolerate both shapes. class EvaluationRun < ApplicationRecord belongs_to :evaluation + # The version of the evaluated agent this run scored, so a pass rate is + # a statement about a release rather than about "the agent". + belongs_to :agent_version, optional: true + before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id } has_many :scenario_results, class_name: "EvaluationScenarioResult", dependent: :destroy enum :status, { pending: 0, running: 1, complete: 2, failed: 3 } diff --git a/actionagent/app/models/action_agent/telemetry_trace.rb b/actionagent/app/models/action_agent/telemetry_trace.rb index 09ca5abb..710d93e8 100644 --- a/actionagent/app/models/action_agent/telemetry_trace.rb +++ b/actionagent/app/models/action_agent/telemetry_trace.rb @@ -43,6 +43,8 @@ class TelemetryTrace < ::ActiveRecord::Base scope :for_date_range, ->(start_date, end_date) { where(timestamp: start_date..end_date) } # The dashboard agent this trace was attributed to on ingest, if any. belongs_to :agent, class_name: "ActionAgent::Agent", optional: true + # The release of the agent this trace came from (see #attach_agent_version!). + belongs_to :agent_version, class_name: "ActionAgent::AgentVersion", optional: true scope :for_account, ->(account) { where(account: account) if ActionAgent.multi_tenant? } @@ -252,7 +254,10 @@ def self.create_from_payload(trace, sdk_info = {}, account: nil, agent: nil) # dashboard-authored agent by guessing a primary key. attrs[:agent_id] = agent&.id - create!(attrs).tap { |record| AgentRegistrar.call(record) } + create!(attrs).tap do |record| + AgentRegistrar.call(record) + record.attach_agent_version! + end end @@ -265,6 +270,28 @@ def self.span_token_sum(span) tokens.fetch("input", 0).to_i + tokens.fetch("output", 0).to_i + tokens.fetch("thinking", 0).to_i end + # Pins this trace to the version of its agent that produced it. The + # instrumentation stamps the root span with `agent.version` — the digest + # ActiveAgent::Release computes from the class — and a release cut on + # deploy carries the same digest, so the two meet here. A trace from a + # dashboard run carries no digest and takes the agent's latest version. + # + # @return [AgentVersion, nil] + def attach_agent_version! + return if agent_version_id.present? || agent_id.blank? + + digest = root_span&.dig("attributes", "agent.version").presence + version = if digest + AgentVersion.find_by(agent_id: agent_id, release_digest: digest) + else + AgentVersion.where(agent_id: agent_id).order(version_number: :desc).first + end + return unless version + + update_columns(agent_version_id: version.id) + version + end + # Returns the root span of this trace. # # @return [Hash, nil] The root span or nil diff --git a/actionagent/app/services/action_agent/agent_registrar.rb b/actionagent/app/services/action_agent/agent_registrar.rb index 16675f4c..91203c43 100644 --- a/actionagent/app/services/action_agent/agent_registrar.rb +++ b/actionagent/app/services/action_agent/agent_registrar.rb @@ -86,6 +86,14 @@ def find_or_create_agent(owner) ) return existing if existing + # A host that mirrors its agent classes into the dashboard (a sync, a + # release) names the class on the record and nothing else: that record + # stands for every action of the class, so a trace from the class is + # its trace — not an observed twin's. Observed records are per action + # and are only ever matched on all three keys above. + mirrored = agents.where.not(status: "observed").find_by(agent_class_name: agent_class) + return mirrored if mirrored + return if agents.observed_agents.count >= MAX_OBSERVED_PER_OWNER create_observed_agent(owner) diff --git a/actionagent/app/services/action_agent/agent_release.rb b/actionagent/app/services/action_agent/agent_release.rb new file mode 100644 index 00000000..94696be1 --- /dev/null +++ b/actionagent/app/services/action_agent/agent_release.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module ActionAgent + # Cuts a version of every dashboard agent that mirrors a host class, from + # that class's release digest — what a deploy runs so the dashboard's + # versions line up with the code that shipped. + # + # The host owns the mirror: whatever syncs its ActiveAgent classes into + # Agent records sets `agent_class_name`, and this reads it back. A record + # whose class no longer resolves, or whose class predates + # ActiveAgent::Release, is reported and skipped rather than failed. + # + # Idempotent: a redeploy of an unchanged agent cuts nothing, so running it + # on every deploy is the intended use (`rake action_agent:agents:release`). + class AgentRelease + Row = Struct.new(:agent, :version, :cut, :skipped, keyword_init: true) + Result = Struct.new(:rows, :revision, keyword_init: true) do + def cut = rows.select(&:cut) + def skipped = rows.select(&:skipped) + end + + # @param revision [String, nil] the deploy (git SHA, release label); + # ActiveAgent::Release.revision when nil + # @param agents [ActiveRecord::Relation] the records to release; every + # record naming a host class by default + # @param released_by [String, nil] recorded on each version cut + def self.call(revision: nil, agents: nil, released_by: nil) + new(revision: revision, agents: agents, released_by: released_by).call + end + + def initialize(revision: nil, agents: nil, released_by: nil) + @revision = revision.presence || ActiveAgent::Release.revision + @agents = agents || Agent.where.not(agent_class_name: [ nil, "" ]) + @released_by = released_by + end + + # @return [Result] + def call + rows = @agents.order(:name).map { |agent| release(agent) } + Result.new(rows: rows, revision: @revision) + end + + private + + def release(agent) + klass = agent.agent_class_name.to_s.safe_constantize + unless klass.respond_to?(:release_digest) + return Row.new(agent: agent, version: nil, cut: false, skipped: "#{agent.agent_class_name} does not resolve to a releasable class") + end + + before = agent.latest_version&.id + version = agent.record_release!( + digest: klass.release_digest, + manifest: klass.release_manifest, + revision: @revision, + released_by: @released_by + ) + Row.new(agent: agent, version: version, cut: version.id != before, skipped: nil) + end + end +end diff --git a/actionagent/lib/generators/action_agent/install_generator.rb b/actionagent/lib/generators/action_agent/install_generator.rb index 6e224c6a..ae894bd1 100644 --- a/actionagent/lib/generators/action_agent/install_generator.rb +++ b/actionagent/lib/generators/action_agent/install_generator.rb @@ -57,6 +57,12 @@ def copy_migrations ) end + # Agent releases arrived after both tables shipped; the migration guards + # every column, so it is emitted for any install that lacks it. + unless existing_migration?("add_agent_releases") + migration_template("add_agent_releases.rb.erb", "db/migrate/add_agent_releases.rb") + end + # The rest of the dashboard — agents, runs, versions, conversations, # evaluations, sandboxes, recordings, keys. Skippable for an app that # only wants to be a trace sink. diff --git a/actionagent/lib/generators/action_agent/templates/add_agent_releases.rb.erb b/actionagent/lib/generators/action_agent/templates/add_agent_releases.rb.erb new file mode 100644 index 00000000..c1e2974b --- /dev/null +++ b/actionagent/lib/generators/action_agent/templates/add_agent_releases.rb.erb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Agent releases: a version cut from the agent's code carries the digest of +# what the model was given and the deploy it shipped in, and every trace, +# run and evaluation run records the version it ran under. Emitted for an +# install whose dashboard tables predate releases; the create-table +# migration carries the same columns for a fresh install. +# +# Each step is guarded so the migration is safe to re-run against a database +# that already has some of these columns. +class AddAgentReleases < ActiveRecord::Migration<%= migration_version %> + def up + add_column_unless_exists :active_agent_agents, :release_digest, :string + + add_column_unless_exists :active_agent_agent_versions, :release_digest, :string + add_column_unless_exists :active_agent_agent_versions, :revision, :string + add_index_unless_exists :active_agent_agent_versions, [ :agent_id, :release_digest ] + + %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table| + add_column_unless_exists table, :agent_version_id, :bigint + add_index_unless_exists table, :agent_version_id + end + end + + def down + remove_column :active_agent_agents, :release_digest if column_exists?(:active_agent_agents, :release_digest) + remove_column :active_agent_agent_versions, :release_digest if column_exists?(:active_agent_agent_versions, :release_digest) + remove_column :active_agent_agent_versions, :revision if column_exists?(:active_agent_agent_versions, :revision) + + %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table| + remove_column table, :agent_version_id if column_exists?(table, :agent_version_id) + end + end + + private + + def add_column_unless_exists(table, column, type) + return unless table_exists?(table) + return if column_exists?(table, column) + + add_column table, column, type + end + + def add_index_unless_exists(table, columns) + return unless table_exists?(table) + return if index_exists?(table, columns) + + add_index table, columns + end +end diff --git a/actionagent/lib/tasks/action_agent.rake b/actionagent/lib/tasks/action_agent.rake index 404b5fd7..03233b31 100644 --- a/actionagent/lib/tasks/action_agent.rake +++ b/actionagent/lib/tasks/action_agent.rake @@ -7,3 +7,36 @@ namespace :action_agent do puts "Agent template library: #{ActionAgent::AgentTemplate.count} templates" end end + +namespace :action_agent do + namespace :agents do + desc "Cut a dashboard version for every agent whose code changed (run on deploy; REVISION= to label it)" + task :release, [ :revision ] => :environment do |_task, args| + Rails.application.eager_load! + result = ActionAgent::AgentRelease.call(revision: args[:revision] || ENV["REVISION"], released_by: ENV["RELEASED_BY"]) + + puts "Release #{result.revision.presence || '(no revision)'}" + result.rows.each do |row| + if row.skipped + puts format(" %-28s skipped — %s", row.agent.name, row.skipped) + elsif row.cut + puts format(" %-28s v%-3d cut %s", row.agent.name, row.version.version_number, row.version.change_summary) + else + puts format(" %-28s v%-3d unchanged (%s)", row.agent.name, row.version.version_number, row.version.release_digest) + end + end + puts "#{result.cut.size} cut, #{result.rows.size - result.cut.size - result.skipped.size} unchanged, #{result.skipped.size} skipped" + end + + desc "List each agent's current version and release" + task versions: :environment do + ActionAgent::Agent.order(:name).find_each do |agent| + version = agent.latest_version + release = agent.latest_release + puts format("%-28s v%-3s %s%s", agent.name, version&.version_number || "-", + release ? "release #{release.release_digest}" : "no release", + release&.revision.present? ? " · #{release.revision}" : "") + end + end + end +end diff --git a/actionagent/test/agent_release_test.rb b/actionagent/test/agent_release_test.rb new file mode 100644 index 00000000..22541893 --- /dev/null +++ b/actionagent/test/agent_release_test.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +require "test_helper" + +# A release cut from the agent's code, and the version every trace, run and +# evaluation run is pinned to as a result. +class AgentReleaseTest < ActiveSupport::TestCase + class BillingAgent < ApplicationAgent + generate_with :mock + + def ask + prompt(message: "hi") + end + end + + setup do + ActionAgent::TelemetryTrace.delete_all + ActionAgent::Agent.delete_all + BillingAgent.reset_release! + @agent = ActionAgent::Agent.create!( + name: "Billing", provider: "mock", model: "mock-1", agent_class_name: "AgentReleaseTest::BillingAgent" + ) + end + + # A trace is stored by the reporter's thread; make sure nothing of this + # test's is still in flight, or still in the table, when the next one runs. + teardown do + ActiveAgent::Telemetry.tracer.flush + ActionAgent::TelemetryTrace.delete_all + ActionAgent::Agent.delete_all + end + + test "a release cuts one version per digest, carrying the digest and the deploy" do + first = @agent.record_release!(digest: "aaaaaaaaaaaa", manifest: { "model" => "m1" }, revision: "sha-1") + + assert first.release? + assert_equal "aaaaaaaaaaaa", first.release_digest + assert_equal "sha-1", first.revision + assert_equal "aaaaaaaaaaaa", @agent.reload.release_digest + assert_match(/first release/, first.change_summary) + assert_equal({ "model" => "m1" }, first.configuration_snapshot["release"]) + + # A redeploy of the same agent is not a new version. + count = @agent.agent_versions.count + assert_equal first.id, @agent.record_release!(digest: "aaaaaaaaaaaa", manifest: { "model" => "m1" }, revision: "sha-2").id + assert_equal count, @agent.agent_versions.count + + second = @agent.record_release!(digest: "bbbbbbbbbbbb", manifest: { "model" => "m2" }, revision: "sha-3") + + assert_equal first.version_number + 1, second.version_number + assert_match(/model/, second.change_summary) + assert_equal second, @agent.latest_release + end + + test "AgentRelease cuts from the host class and skips a record whose class is not releasable" do + ghost = ActionAgent::Agent.create!(name: "Ghost", provider: "mock", model: "m", agent_class_name: "Nope::Missing") + + result = ActionAgent::AgentRelease.call(revision: "deploy-1") + row = result.rows.find { |r| r.agent == @agent } + + assert row.cut + assert_equal BillingAgent.release_digest, row.version.release_digest + assert_equal "deploy-1", row.version.revision + assert result.rows.find { |r| r.agent == ghost }.skipped + + assert_not ActionAgent::AgentRelease.call(revision: "deploy-2").rows.find { |r| r.agent == @agent }.cut + end + + test "an ingested trace is pinned to the release its root span names" do + version = @agent.record_release!(digest: "cccccccccccc") + + trace = ActionAgent::TelemetryTrace.create_from_payload(payload("agent.version" => "cccccccccccc"), {}, agent: @agent) + + assert_equal version.id, trace.reload.agent_version_id + assert_equal version, trace.agent_version + end + + test "a trace without a digest takes the agent's latest version, and an unknown digest none" do + version = @agent.record_release!(digest: "dddddddddddd") + + assert_equal version.id, ActionAgent::TelemetryTrace.create_from_payload(payload, {}, agent: @agent).reload.agent_version_id + assert_nil ActionAgent::TelemetryTrace.create_from_payload(payload("agent.version" => "eeeeeeeeeeee"), {}, agent: @agent).reload.agent_version_id + end + + test "runs and evaluation runs record the version they executed under" do + version = @agent.record_release!(digest: "ffffffffffff") + + run = @agent.agent_runs.create!(input_prompt: "hi", action_name: "ask", trace_id: SecureRandom.uuid) + assert_equal version.id, run.agent_version_id + + evaluation = ActionAgent::Evaluation.create!( + agent: @agent, name: "suite", criteria: [ { "key" => "present", "type" => "response_present" } ] + ) + assert_equal version.id, evaluation.evaluation_runs.create!(status: :pending).agent_version_id + end + + test "a generation from the mirrored class is attributed to the mirror, not to an observed twin" do + config = ActiveAgent::Telemetry.configuration + saved = { enabled: config.enabled, local_storage: config.local_storage } + config.enabled = true + config.local_storage = true + instrument(BillingAgent) + + BillingAgent.ask.generate_now + ActiveAgent::Telemetry.tracer.flush + + assert_equal @agent.id, ActionAgent::TelemetryTrace.order(:id).last.agent_id + assert_equal 1, ActionAgent::Agent.count, "the trace registered an observed twin instead of matching the mirror" + ensure + config.enabled = saved[:enabled] + config.local_storage = saved[:local_storage] + end + + test "a real generation stamps the release on its trace" do + config = ActiveAgent::Telemetry.configuration + saved = { enabled: config.enabled, local_storage: config.local_storage } + config.enabled = true + config.local_storage = true + instrument(BillingAgent) + ActiveAgent::Release.revision = "rev-9" + version = @agent.record_release!(digest: BillingAgent.release_digest, revision: "rev-9") + + BillingAgent.ask.generate_now + ActiveAgent::Telemetry.tracer.flush + trace = ActionAgent::TelemetryTrace.order(:id).last + + assert trace, "no trace stored" + assert_equal BillingAgent.release_digest, trace.root_span.dig("attributes", "agent.version") + assert_equal "rev-9", trace.root_span.dig("attributes", "agent.revision") + assert_equal "rev-9", trace.root_span.dig("attributes", "service.version") + assert_equal version.id, trace.agent_version_id + ensure + ActiveAgent::Release.revision = nil + config.enabled = saved[:enabled] + config.local_storage = saved[:local_storage] + end + + private + + # The dummy boots with telemetry off, so the railtie never installed the + # generation instrumentation. Install it on this test's class only: on + # ActiveAgent::Base it would stay for the rest of the process, and every + # later test that enables telemetry would trace its generations and + # register observed agents that leak into unrelated tests. + def instrument(klass) + klass.include(ActiveAgent::Telemetry::Instrumentation) + klass.instrument_telemetry! + end + + def payload(attributes = {}) + { + "trace_id" => SecureRandom.uuid, "service_name" => "dummy", "environment" => "test", + "timestamp" => Time.current.iso8601(6), + "spans" => [ { + "span_id" => "r1", "parent_span_id" => nil, "name" => "BillingAgent.ask", "type" => "root", + "duration_ms" => 10.0, "status" => "OK", + "attributes" => { "agent.class" => "AgentReleaseTest::BillingAgent", "agent.action" => "ask" }.merge(attributes) + } ] + } + end +end diff --git a/lib/active_agent/base.rb b/lib/active_agent/base.rb index 702d9d78..b6bb826c 100644 --- a/lib/active_agent/base.rb +++ b/lib/active_agent/base.rb @@ -12,6 +12,7 @@ require "active_agent/concerns/preview" require "active_agent/concerns/provider" require "active_agent/concerns/queueing" +require "active_agent/concerns/release" require "active_agent/concerns/rescue" require "active_agent/concerns/streaming" require "active_agent/concerns/tooling" @@ -49,6 +50,7 @@ class Base < AbstractController::Base include Parameterized include Provider include Queueing + include Release include Rescue include Streaming include Tooling diff --git a/lib/active_agent/concerns/release.rb b/lib/active_agent/concerns/release.rb new file mode 100644 index 00000000..c83e0395 --- /dev/null +++ b/lib/active_agent/concerns/release.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +require "digest" +require "json" + +module ActiveAgent + # A release of an agent is what the model is given: the provider and model, + # the generation options, the prompt templates on disk, the actions and the + # tools the class declares. {ClassMethods#release_digest} names that + # deterministically, so two deploys that ship the same agent share a digest + # and a change to any of those inputs yields a new one — without anyone + # bumping a number by hand. + # + # The digest is what telemetry stamps on every generation (`agent.version`) + # and what a dashboard cuts an AgentVersion from on deploy, so a trace, a + # run and an evaluation can all say which release of the agent produced + # them. {Release.revision} carries the deploy itself (a git SHA or a release + # label) alongside, when the host knows it. + # + # Class-level and memoized: in development a reload replaces the class, so + # the next reference recomputes it. + module Release + extend ActiveSupport::Concern + + # Option keys that never belong in a manifest: credentials, and per-call + # state the class does not own. + EXCLUDED_OPTION_KEYS = %i[ + api_key access_token secret password token trace_id messages message instructions + ].freeze + SECRET_KEY_PATTERN = /key|token|secret|password|credential/i + + # How the deploy identifies itself, when it does. A host sets + # `ActiveAgent::Release.revision = ENV["GIT_SHA"]` (or a proc) from an + # initializer; otherwise the conventional deploy variables are read. + class << self + attr_writer :revision + + # @return [String, nil] + def revision + value = @revision.respond_to?(:call) ? @revision.call : @revision + value = value.presence || ENV.values_at("SERVICE_VERSION", "GIT_SHA", "KAMAL_VERSION", "SOURCE_VERSION", "HEROKU_SLUG_COMMIT").find(&:present?) + value&.to_s + end + + # Canonical JSON: sorted keys at every level, so the digest does not + # depend on the order anything was declared in. + # @api private + def canonical(value) + case value + when Hash then value.map { |k, v| [ k.to_s, canonical(v) ] }.sort_by(&:first).to_h + when Array then value.map { |v| canonical(v) } + when Symbol then value.to_s + else value + end + end + end + + class_methods do + # Everything about this class that shapes a generation, as data. + # + # @return [Hash] + def release_manifest + @release_manifest ||= Release.canonical( + agent: name, + provider: release_provider, + model: prompt_options&.dig(:model), + options: release_options, + actions: release_actions, + templates: release_templates, + tools: release_tools, + delegations: release_delegations + ) + end + + # A short, stable identifier for {#release_manifest}: the first twelve + # hex characters of its SHA-256. + # + # @return [String] + def release_digest + @release_digest ||= Digest::SHA256.hexdigest(JSON.generate(release_manifest))[0, 12] + end + + # Forgets the memoized manifest and digest — for a host that edits + # templates at runtime, and for tests. + # @return [void] + def reset_release! + @release_manifest = nil + @release_digest = nil + end + + private + + def release_provider + provider = respond_to?(:prompt_provider) ? prompt_provider : nil + (provider || prompt_options&.dig(:service))&.to_s + end + + # Generation options minus credentials and per-call state. Nested + # hashes are walked so a token under `options: { headers: … }` is + # dropped too. + def release_options + strip_secrets((prompt_options || {}).except(*EXCLUDED_OPTION_KEYS, :model, :service)) + end + + def strip_secrets(value) + case value + when Hash + value.each_with_object({}) do |(key, inner), kept| + next if EXCLUDED_OPTION_KEYS.include?(key.to_sym) || key.to_s.match?(SECRET_KEY_PATTERN) + + kept[key] = strip_secrets(inner) + end + when Array then value.map { |inner| strip_secrets(inner) } + else value + end + end + + # The public actions — the prompts a caller can invoke. + def release_actions + respond_to?(:action_methods) ? action_methods.to_a.sort : [] + end + + # Every template file under this agent's view prefixes, keyed by its + # path relative to the view root, with a digest of its contents. The + # prefixes mirror View#_prefixes without an action: `app/views//` + # and `app/views/agents//`. + def release_templates + return {} if anonymous? || !respond_to?(:view_paths) + + base = name.underscore + prefixes = [ base, "agents/#{base.delete_suffix("_agent")}" ] + roots = Array(view_paths).map { |path| path.respond_to?(:to_path) ? path.to_path : path.to_s } + + roots.each_with_object({}) do |root, templates| + prefixes.each do |prefix| + Dir.glob(File.join(root, prefix, "**", "*")).sort.each do |file| + next unless File.file?(file) + + relative = file.delete_prefix("#{root}/") + templates[relative] = Digest::SHA256.hexdigest(File.binread(file))[0, 12] + end + end + end + end + + # Tool definitions the class declares itself (a host convention such as + # schema-derived rosters), reduced to what identifies them. + def release_tools + return [] unless respond_to?(:tool_definitions) + + Array(tool_definitions).map do |definition| + next definition.to_s unless definition.respond_to?(:to_h) + + hash = definition.to_h + { + name: (hash[:name] || hash["name"]).to_s, + description: (hash[:description] || hash["description"]).to_s, + parameters: hash[:parameters] || hash["parameters"] + } + end.sort_by { |tool| tool.is_a?(Hash) ? tool[:name] : tool } + end + + # The delegations this class declares, by tool name. + def release_delegations + return [] unless respond_to?(:delegations) + + Array(delegations).map { |tool_name, _definition| tool_name.to_s }.sort + end + end + end +end diff --git a/lib/active_agent/telemetry/configuration.rb b/lib/active_agent/telemetry/configuration.rb index d57df80d..73151df8 100644 --- a/lib/active_agent/telemetry/configuration.rb +++ b/lib/active_agent/telemetry/configuration.rb @@ -24,6 +24,17 @@ class Configuration < ActiveAgents::Telemetry::Configuration # @return [Boolean] Whether to store traces in the app's own database attr_reader :local_storage + # The deploy every trace is stamped with (`service.version`). Unset, it + # is whatever ActiveAgent::Release.revision resolves — a git SHA from + # the conventional deploy variables — so traces from two deploys of + # the same service can be told apart without any host configuration. + attr_writer :service_version + + def service_version + value = @service_version.respond_to?(:call) ? @service_version.call : @service_version + (value.presence || ActiveAgent::Release.revision)&.to_s + end + def initialize super # The framework predates the shared gem and has always been opt-in; diff --git a/lib/active_agent/telemetry/instrumentation.rb b/lib/active_agent/telemetry/instrumentation.rb index 8eb4dcd1..7578f686 100644 --- a/lib/active_agent/telemetry/instrumentation.rb +++ b/lib/active_agent/telemetry/instrumentation.rb @@ -58,6 +58,15 @@ def process_prompt span.set_attribute("agent.action", action_name.to_s) span.set_attribute("agent.provider", provider_name) span.set_attribute("agent.model", model_name) + # Which release of the agent ran: the digest of what the model was + # given (ActiveAgent::Release), so a dashboard can pin this trace + # to the version it cut on deploy. + if self.class.respond_to?(:release_digest) + span.set_attribute("agent.version", self.class.release_digest) + if (revision = ActiveAgent::Release.revision).present? + span.set_attribute("agent.revision", revision) + end + end # Add prompt span, carrying the prompt contents (instructions + # outbound messages) so dashboards can show what was sent. diff --git a/lib/active_agent/telemetry/tracer.rb b/lib/active_agent/telemetry/tracer.rb index 1201c3fb..6f25a2aa 100644 --- a/lib/active_agent/telemetry/tracer.rb +++ b/lib/active_agent/telemetry/tracer.rb @@ -169,12 +169,15 @@ def should_trace? # Returns default attributes for all spans. def default_attributes - { + attributes = { "service.name" => configuration.resolved_service_name, "service.environment" => configuration.resolved_environment, "telemetry.sdk.name" => "activeagent", "telemetry.sdk.version" => ActiveAgent::VERSION } + version = configuration.respond_to?(:service_version) ? configuration.service_version : nil + attributes["service.version"] = version if version.present? + attributes end end end diff --git a/test/dummy/db/migrate/009_add_agent_releases.rb b/test/dummy/db/migrate/009_add_agent_releases.rb new file mode 100644 index 00000000..6c3b8d6e --- /dev/null +++ b/test/dummy/db/migrate/009_add_agent_releases.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Agent releases: a version cut from the agent's code carries the digest of +# what the model was given and the deploy it shipped in, and every trace, +# run and evaluation run records the version it ran under. Emitted for an +# install whose dashboard tables predate releases; the create-table +# migration carries the same columns for a fresh install. +# +# Each step is guarded so the migration is safe to re-run against a database +# that already has some of these columns. +class AddAgentReleases < ActiveRecord::Migration[7.2] + def up + add_column_unless_exists :active_agent_agents, :release_digest, :string + + add_column_unless_exists :active_agent_agent_versions, :release_digest, :string + add_column_unless_exists :active_agent_agent_versions, :revision, :string + add_index_unless_exists :active_agent_agent_versions, [ :agent_id, :release_digest ] + + %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table| + add_column_unless_exists table, :agent_version_id, :bigint + add_index_unless_exists table, :agent_version_id + end + end + + def down + remove_column :active_agent_agents, :release_digest if column_exists?(:active_agent_agents, :release_digest) + remove_column :active_agent_agent_versions, :release_digest if column_exists?(:active_agent_agent_versions, :release_digest) + remove_column :active_agent_agent_versions, :revision if column_exists?(:active_agent_agent_versions, :revision) + + %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table| + remove_column table, :agent_version_id if column_exists?(table, :agent_version_id) + end + end + + private + + def add_column_unless_exists(table, column, type) + return unless table_exists?(table) + return if column_exists?(table, column) + + add_column table, column, type + end + + def add_index_unless_exists(table, columns) + return unless table_exists?(table) + return if index_exists?(table, columns) + + add_index table, columns + end +end diff --git a/test/dummy/db/schema.rb b/test/dummy/db/schema.rb index 39819837..b9452945 100644 --- a/test/dummy/db/schema.rb +++ b/test/dummy/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 8) do +ActiveRecord::Schema[8.0].define(version: 9) do create_table "active_agent_agent_contexts", force: :cascade do |t| t.string "action_name", null: false t.string "agent_name", null: false @@ -85,6 +85,8 @@ end create_table "active_agent_agent_runs", force: :cascade do |t| + t.bigint "agent_version_id" + t.index [ "agent_version_id" ], name: "index_active_agent_agent_runs_on_agent_version_id" t.string "action_name" t.bigint "agent_id", null: false t.datetime "completed_at" @@ -134,6 +136,9 @@ end create_table "active_agent_agent_versions", force: :cascade do |t| + t.string "release_digest" + t.string "revision" + t.index [ "agent_id", "release_digest" ], name: "index_active_agent_agent_versions_on_agent_id_and_release_digest" t.bigint "agent_id", null: false t.string "change_summary" t.json "configuration_snapshot", default: {}, null: false @@ -145,6 +150,7 @@ end create_table "active_agent_agents", force: :cascade do |t| + t.string "release_digest" t.bigint "account_id" t.string "action_name" t.json "action_prompts", default: [], null: false @@ -189,6 +195,8 @@ end create_table "active_agent_evaluation_runs", force: :cascade do |t| + t.bigint "agent_version_id" + t.index [ "agent_version_id" ], name: "index_active_agent_evaluation_runs_on_agent_version_id" t.datetime "completed_at" t.datetime "created_at", null: false t.text "error_message" @@ -356,6 +364,8 @@ end create_table "active_agent_telemetry_traces", force: :cascade do |t| + t.bigint "agent_version_id" + t.index [ "agent_version_id" ], name: "index_active_agent_telemetry_traces_on_agent_version_id" t.string "agent_action" t.string "agent_class" t.bigint "agent_id" diff --git a/test/release_test.rb b/test/release_test.rb new file mode 100644 index 00000000..02cafb32 --- /dev/null +++ b/test/release_test.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" + +# A release is the digest of what the model is given. These pin what goes +# into it, what stays out, and that a change to any input is a new digest. +class ReleaseTest < ActiveSupport::TestCase + class BillingAgent < ApplicationAgent + generate_with :mock, temperature: 0.2, api_key: "sk-not-for-the-manifest" + + def summarize + prompt(message: "summarize") + end + end + + class OtherAgent < ApplicationAgent + generate_with :mock + + def summarize + prompt(message: "summarize") + end + end + + setup do + BillingAgent.reset_release! + OtherAgent.reset_release! + end + + teardown { ActiveAgent::Release.revision = nil } + + test "the manifest names what the model is given, without credentials" do + manifest = BillingAgent.release_manifest + + assert_equal "ReleaseTest::BillingAgent", manifest["agent"] + assert manifest.key?("provider") + assert_includes manifest["actions"], "summarize" + assert_equal 0.2, manifest["options"]["temperature"] + assert_not manifest["options"].key?("api_key") + assert_no_match(/sk-not-for-the-manifest/, JSON.generate(manifest)) + end + + test "the digest is short, stable, and differs between agents" do + assert_match(/\A\h{12}\z/, BillingAgent.release_digest) + assert_equal BillingAgent.release_digest, BillingAgent.release_digest + assert_not_equal BillingAgent.release_digest, OtherAgent.release_digest + end + + test "a change to a prompt template is a new release" do + Dir.mktmpdir do |root| + dir = File.join(root, "release_test/billing_agent") + FileUtils.mkdir_p(dir) + File.write(File.join(dir, "instructions.md.erb"), "Be brief.") + BillingAgent.view_paths = [ root ] + BillingAgent.reset_release! + + before = BillingAgent.release_digest + assert_equal [ "release_test/billing_agent/instructions.md.erb" ], BillingAgent.release_manifest["templates"].keys + + File.write(File.join(dir, "instructions.md.erb"), "Be thorough.") + BillingAgent.reset_release! + + assert_not_equal before, BillingAgent.release_digest + ensure + BillingAgent.view_paths = ApplicationAgent.view_paths + BillingAgent.reset_release! + end + end + + test "the revision is configured, computed, or read from the deploy environment" do + ActiveAgent::Release.revision = "abc1234" + assert_equal "abc1234", ActiveAgent::Release.revision + + ActiveAgent::Release.revision = -> { "def5678" } + assert_equal "def5678", ActiveAgent::Release.revision + + ActiveAgent::Release.revision = nil + with_env("GIT_SHA" => "0123abc") { assert_equal "0123abc", ActiveAgent::Release.revision } + end + + test "the telemetry service version follows the release revision unless set" do + config = ActiveAgent::Telemetry.configuration + ActiveAgent::Release.revision = "rev-1" + assert_equal "rev-1", config.service_version + + config.service_version = "2026.09" + assert_equal "2026.09", config.service_version + ensure + config.service_version = nil + end + + private + + def with_env(values) + previous = values.keys.to_h { |key| [ key, ENV[key] ] } + values.each { |key, value| ENV[key] = value } + yield + ensure + previous.each { |key, value| value.nil? ? ENV.delete(key) : ENV[key] = value } + end +end