feat(agents): releases — a digest of what the model is given, cut on deploy, pinned to every trace, run and evaluation

CHANGELOG.md

@@ -9,6 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- **Agents have releases, and every trace, run and evaluation says which one
+  it ran under.** `ActiveAgent::Release` gives each agent class a digest of
+  what the model is given — provider and model, generation options minus
+  credentials, the actions, the prompt templates on disk, and the tools and
+  delegations it declares — so two deploys of the same agent share a digest
+  and any change to those inputs is a new one, with no number to bump.
+  `ActiveAgent::Release.revision` carries the deploy alongside (a git SHA;
+  read from `SERVICE_VERSION`, `GIT_SHA`, `KAMAL_VERSION` and friends when
+  not set). The instrumentation stamps `agent.version` and `agent.revision`
+  on every generation's root span, and every trace gets `service.version`.
+  In the dashboard, `rake action_agent:agents:release[REVISION]` cuts an
+  `AgentVersion` for each agent whose code changed since the last release —
+  idempotent, so it belongs in the deploy — `rake action_agent:agents:versions`
+  lists them, and `Agent#record_release!` is the call behind both for a host
+  that syncs agents its own way. Traces are pinned to the release their root
+  span names, runs and evaluation runs to the version current when they
+  started (`agent_version_id` on all three; the install generator emits the
+  migration). A version's JSON carries `release`, `release_digest` and
+  `revision`, so the Versions tab tells a deploy from an edit. For that to
+  reach a host's own agents, a trace from a class the host mirrors into the
+  dashboard is now attributed to that mirror — the registrar matched only on
+  service, class *and* action, so every code-path trace registered an
+  observed per-action twin beside the synced record and could never be
+  pinned to its release.
 - **An evaluation replay runs as the evaluation's owner.** The scenario runner
   handed `Agent#test_execute` no caller, so every tool a replay called ran
   unattributed and a host scope answered empty — the suite graded an agent

actionagent/app/controllers/action_agent/api/agents_controller.rb

@@ -532,7 +532,10 @@ def version_json(version, include_diff: false)
           change_summary: version.change_summary,
           created_by: version.created_by,
           created_at: version.created_at,
-          is_latest: version.latest?
+          is_latest: version.latest?,
+          release: version.release?,
+          release_digest: version.release_digest,
+          revision: version.revision
         }
 
         if include_diff && version.previous

actionagent/app/models/action_agent/agent.rb

@@ -219,6 +219,46 @@ def latest_version
       agent_versions.order(version_number: :desc).first
     end
 
+    # The most recent version cut from the agent's code, if any.
+    # @return [AgentVersion, nil]
+    def latest_release
+      agent_versions.releases.order(version_number: :desc).first
+    end
+
+    # Cuts a version for a release of the agent's code, identified by the
+    # digest ActiveAgent::Release computes from what the model is given.
+    # Returns the existing version when the latest release already carries
+    # this digest — a redeploy of an unchanged agent is not a new version —
+    # so it is safe to call on every deploy.
+    #
+    # The version's snapshot is the dashboard configuration plus the release
+    # manifest under "release", so the Versions tab can diff two releases the
+    # same way it diffs two dashboard edits.
+    #
+    # @param digest [String] ActiveAgent::Release digest of the host class
+    # @param manifest [Hash, nil] the class's release manifest
+    # @param revision [String, nil] the deploy (git SHA, release label)
+    # @param released_by [String, nil]
+    # @return [AgentVersion]
+    def record_release!(digest:, manifest: nil, revision: nil, released_by: nil)
+      current = latest_release
+      if current && current.release_digest == digest
+        update_columns(release_digest: digest) if release_digest != digest
+        return current
+      end
+
+      version = agent_versions.create!(
+        version_number: (latest_version&.version_number || 0) + 1,
+        change_summary: release_summary(digest, revision, current&.configuration_snapshot&.dig("release"), manifest),
+        configuration_snapshot: configuration_snapshot.merge("release" => manifest || {}),
+        release_digest: digest,
+        revision: revision,
+        created_by: released_by || "release"
+      )
+      update_columns(release_digest: digest)
+      version
+    end
+
     # Maps each historical instructions digest to the first version that
     # introduced it ("v3"), so run cohorts can label instruction changes with
     # real agent versions instead of raw hashes.
@@ -396,6 +436,18 @@ def generate_slug
       end
     end
 
+    # "Release 1a2b3c4d5e6f · abc1234: templates, tools" — the digest, the
+    # deploy, and which parts of the manifest moved since the last release.
+    def release_summary(digest, revision, previous_manifest, manifest)
+      label = [ "Release #{digest}", revision.presence ].compact.join(" · ")
+      return "#{label}: first release" if previous_manifest.blank? || manifest.blank?
+
+      changed = (previous_manifest.keys | manifest.stringify_keys.keys).select do |key|
+        previous_manifest[key] != manifest.stringify_keys[key]
+      end
+      changed.any? ? "#{label}: #{changed.sort.join(', ')}" : label
+    end
+
     def create_initial_version
       agent_versions.create!(
         version_number: 1,

actionagent/app/models/action_agent/agent_run.rb

@@ -3,6 +3,10 @@
 module ActionAgent
   class AgentRun < ApplicationRecord
     belongs_to :agent
+    # The version of the agent this run executed under — the latest at the
+    # time, since a run is against the agent as it is.
+    belongs_to :agent_version, optional: true
+    before_create { self.agent_version_id ||= agent&.latest_version&.id }
 
     # Raised when a caller hands a run files to attach in a host app that
     # has nowhere to keep them.

actionagent/app/models/action_agent/agent_version.rb

@@ -9,6 +9,9 @@ class AgentVersion < ApplicationRecord
 
     # Scopes
     scope :recent, -> { order(version_number: :desc) }
+    # Versions cut from the agent's code on deploy, as opposed to edits made
+    # in the dashboard.
+    scope :releases, -> { where.not(release_digest: [ nil, "" ]) }
     scope :by_version, ->(num) { where(version_number: num) }
 
     # Compare two versions
@@ -36,6 +39,13 @@ def next_version
     end
 
     # Check if this is the latest version
+    # Whether this version was cut from the agent's code (it carries the
+    # release digest) rather than from a dashboard edit.
+    # @return [Boolean]
+    def release?
+      release_digest.present?
+    end
+
     def latest?
       agent.latest_version&.id == id
     end

actionagent/app/models/action_agent/evaluation_run.rb

@@ -8,6 +8,10 @@ module ActionAgent
   # See #average_score, which is what has to tolerate both shapes.
   class EvaluationRun < ApplicationRecord
     belongs_to :evaluation
+    # The version of the evaluated agent this run scored, so a pass rate is
+    # a statement about a release rather than about "the agent".
+    belongs_to :agent_version, optional: true
+    before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id }
     has_many :scenario_results, class_name: "EvaluationScenarioResult", dependent: :destroy
 
     enum :status, { pending: 0, running: 1, complete: 2, failed: 3 }

actionagent/app/models/action_agent/telemetry_trace.rb

@@ -43,6 +43,8 @@ class TelemetryTrace < ::ActiveRecord::Base
     scope :for_date_range, ->(start_date, end_date) { where(timestamp: start_date..end_date) }
     # The dashboard agent this trace was attributed to on ingest, if any.
     belongs_to :agent, class_name: "ActionAgent::Agent", optional: true
+    # The release of the agent this trace came from (see #attach_agent_version!).
+    belongs_to :agent_version, class_name: "ActionAgent::AgentVersion", optional: true
 
     scope :for_account, ->(account) { where(account: account) if ActionAgent.multi_tenant? }
 
@@ -252,7 +254,10 @@ def self.create_from_payload(trace, sdk_info = {}, account: nil, agent: nil)
       # dashboard-authored agent by guessing a primary key.
       attrs[:agent_id] = agent&.id
 
-      create!(attrs).tap { |record| AgentRegistrar.call(record) }
+      create!(attrs).tap do |record|
+        AgentRegistrar.call(record)
+        record.attach_agent_version!
+      end
     end
 
 
@@ -265,6 +270,28 @@ def self.span_token_sum(span)
       tokens.fetch("input", 0).to_i + tokens.fetch("output", 0).to_i + tokens.fetch("thinking", 0).to_i
     end
 
+    # Pins this trace to the version of its agent that produced it. The
+    # instrumentation stamps the root span with `agent.version` — the digest
+    # ActiveAgent::Release computes from the class — and a release cut on
+    # deploy carries the same digest, so the two meet here. A trace from a
+    # dashboard run carries no digest and takes the agent's latest version.
+    #
+    # @return [AgentVersion, nil]
+    def attach_agent_version!
+      return if agent_version_id.present? || agent_id.blank?
+
+      digest = root_span&.dig("attributes", "agent.version").presence
+      version = if digest
+        AgentVersion.find_by(agent_id: agent_id, release_digest: digest)
+      else
+        AgentVersion.where(agent_id: agent_id).order(version_number: :desc).first
+      end
+      return unless version
+
+      update_columns(agent_version_id: version.id)
+      version
+    end
+
     # Returns the root span of this trace.
     #
     # @return [Hash, nil] The root span or nil

actionagent/app/services/action_agent/agent_registrar.rb

@@ -86,6 +86,14 @@ def find_or_create_agent(owner)
       )
       return existing if existing
 
+      # A host that mirrors its agent classes into the dashboard (a sync, a
+      # release) names the class on the record and nothing else: that record
+      # stands for every action of the class, so a trace from the class is
+      # its trace — not an observed twin's. Observed records are per action
+      # and are only ever matched on all three keys above.
+      mirrored = agents.where.not(status: "observed").find_by(agent_class_name: agent_class)
+      return mirrored if mirrored
+
       return if agents.observed_agents.count >= MAX_OBSERVED_PER_OWNER
 
       create_observed_agent(owner)

actionagent/app/services/action_agent/agent_release.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Cuts a version of every dashboard agent that mirrors a host class, from
+  # that class's release digest — what a deploy runs so the dashboard's
+  # versions line up with the code that shipped.
+  #
+  # The host owns the mirror: whatever syncs its ActiveAgent classes into
+  # Agent records sets `agent_class_name`, and this reads it back. A record
+  # whose class no longer resolves, or whose class predates
+  # ActiveAgent::Release, is reported and skipped rather than failed.
+  #
+  # Idempotent: a redeploy of an unchanged agent cuts nothing, so running it
+  # on every deploy is the intended use (`rake action_agent:agents:release`).
+  class AgentRelease
+    Row = Struct.new(:agent, :version, :cut, :skipped, keyword_init: true)
+    Result = Struct.new(:rows, :revision, keyword_init: true) do
+      def cut = rows.select(&:cut)
+      def skipped = rows.select(&:skipped)
+    end
+
+    # @param revision [String, nil] the deploy (git SHA, release label);
+    #   ActiveAgent::Release.revision when nil
+    # @param agents [ActiveRecord::Relation] the records to release; every
+    #   record naming a host class by default
+    # @param released_by [String, nil] recorded on each version cut
+    def self.call(revision: nil, agents: nil, released_by: nil)
+      new(revision: revision, agents: agents, released_by: released_by).call
+    end
+
+    def initialize(revision: nil, agents: nil, released_by: nil)
+      @revision = revision.presence || ActiveAgent::Release.revision
+      @agents = agents || Agent.where.not(agent_class_name: [ nil, "" ])
+      @released_by = released_by
+    end
+
+    # @return [Result]
+    def call
+      rows = @agents.order(:name).map { |agent| release(agent) }
+      Result.new(rows: rows, revision: @revision)
+    end
+
+    private
+
+    def release(agent)
+      klass = agent.agent_class_name.to_s.safe_constantize
+      unless klass.respond_to?(:release_digest)
+        return Row.new(agent: agent, version: nil, cut: false, skipped: "#{agent.agent_class_name} does not resolve to a releasable class")
+      end
+
+      before = agent.latest_version&.id
+      version = agent.record_release!(
+        digest: klass.release_digest,
+        manifest: klass.release_manifest,
+        revision: @revision,
+        released_by: @released_by
+      )
+      Row.new(agent: agent, version: version, cut: version.id != before, skipped: nil)
+    end
+  end
+end

actionagent/lib/generators/action_agent/install_generator.rb

@@ -57,6 +57,12 @@ def copy_migrations
         )
       end
 
+      # Agent releases arrived after both tables shipped; the migration guards
+      # every column, so it is emitted for any install that lacks it.
+      unless existing_migration?("add_agent_releases")
+        migration_template("add_agent_releases.rb.erb", "db/migrate/add_agent_releases.rb")
+      end
+
       # The rest of the dashboard — agents, runs, versions, conversations,
       # evaluations, sandboxes, recordings, keys. Skippable for an app that
       # only wants to be a trace sink.

actionagent/lib/generators/action_agent/templates/add_agent_releases.rb.erb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Agent releases: a version cut from the agent's code carries the digest of
+# what the model was given and the deploy it shipped in, and every trace,
+# run and evaluation run records the version it ran under. Emitted for an
+# install whose dashboard tables predate releases; the create-table
+# migration carries the same columns for a fresh install.
+#
+# Each step is guarded so the migration is safe to re-run against a database
+# that already has some of these columns.
+class AddAgentReleases < ActiveRecord::Migration<%= migration_version %>
+  def up
+    add_column_unless_exists :active_agent_agents, :release_digest, :string
+
+    add_column_unless_exists :active_agent_agent_versions, :release_digest, :string
+    add_column_unless_exists :active_agent_agent_versions, :revision, :string
+    add_index_unless_exists :active_agent_agent_versions, [ :agent_id, :release_digest ]
+
+    %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table|
+      add_column_unless_exists table, :agent_version_id, :bigint
+      add_index_unless_exists table, :agent_version_id
+    end
+  end
+
+  def down
+    remove_column :active_agent_agents, :release_digest if column_exists?(:active_agent_agents, :release_digest)
+    remove_column :active_agent_agent_versions, :release_digest if column_exists?(:active_agent_agent_versions, :release_digest)
+    remove_column :active_agent_agent_versions, :revision if column_exists?(:active_agent_agent_versions, :revision)
+
+    %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table|
+      remove_column table, :agent_version_id if column_exists?(table, :agent_version_id)
+    end
+  end
+
+  private
+
+  def add_column_unless_exists(table, column, type)
+    return unless table_exists?(table)
+    return if column_exists?(table, column)
+
+    add_column table, column, type
+  end
+
+  def add_index_unless_exists(table, columns)
+    return unless table_exists?(table)
+    return if index_exists?(table, columns)
+
+    add_index table, columns
+  end
+end

actionagent/lib/tasks/action_agent.rake

@@ -7,3 +7,36 @@ namespace :action_agent do
     puts "Agent template library: #{ActionAgent::AgentTemplate.count} templates"
   end
 end
+
+namespace :action_agent do
+  namespace :agents do
+    desc "Cut a dashboard version for every agent whose code changed (run on deploy; REVISION=<git sha> to label it)"
+    task :release, [ :revision ] => :environment do |_task, args|
+      Rails.application.eager_load!
+      result = ActionAgent::AgentRelease.call(revision: args[:revision] || ENV["REVISION"], released_by: ENV["RELEASED_BY"])
+
+      puts "Release #{result.revision.presence || '(no revision)'}"
+      result.rows.each do |row|
+        if row.skipped
+          puts format("  %-28s skipped — %s", row.agent.name, row.skipped)
+        elsif row.cut
+          puts format("  %-28s v%-3d cut   %s", row.agent.name, row.version.version_number, row.version.change_summary)
+        else
+          puts format("  %-28s v%-3d unchanged (%s)", row.agent.name, row.version.version_number, row.version.release_digest)
+        end
+      end
+      puts "#{result.cut.size} cut, #{result.rows.size - result.cut.size - result.skipped.size} unchanged, #{result.skipped.size} skipped"
+    end
+
+    desc "List each agent's current version and release"
+    task versions: :environment do
+      ActionAgent::Agent.order(:name).find_each do |agent|
+        version = agent.latest_version
+        release = agent.latest_release
+        puts format("%-28s v%-3s %s%s", agent.name, version&.version_number || "-",
+                    release ? "release #{release.release_digest}" : "no release",
+                    release&.revision.present? ? " · #{release.revision}" : "")
+      end
+    end
+  end
+end