From e4e3dcce280d734bd06d2036ce009e8bef8d18cd Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 18:22:33 +0100 Subject: [PATCH 1/7] fix(subagents): retain possible-effects observations on failure --- docs/issues/possible-effects.md | 41 +++++++++ src/acp_child.rs | 145 +++++++++++++++++++++++++----- src/effects.rs | 154 ++++++++++++++++++++++++++++++++ src/fatal.rs | 89 +++++++++++++++++- src/lib.rs | 1 + src/tools/subagent.rs | 92 +++++++++++++------ src/tools/subagent/tests.rs | 131 ++++++++++++++++++++++++++- 7 files changed, 600 insertions(+), 53 deletions(-) create mode 100644 docs/issues/possible-effects.md create mode 100644 src/effects.rs diff --git a/docs/issues/possible-effects.md b/docs/issues/possible-effects.md new file mode 100644 index 00000000..0ff9f1eb --- /dev/null +++ b/docs/issues/possible-effects.md @@ -0,0 +1,41 @@ +# Possible-effects observations (issue #48) + +A failed child prompt can have changed external state. Kit retains bounded, +provider-independent positive observations from ACP notifications, independently +of its output-retention limits and opt-in stderr display events. The owner of the +observations outlives the dispatched prompt task, so reply-channel loss does not +silently discard previously observed activity. + +`possible_effects` contains only fixed field names, an allowlisted source, and +booleans. It never contains tool IDs, arguments, results, assistant text, prompts, +or provider payloads. `source: acp_notifications` identifies lifecycle **reports** +from a harness, not verified local execution receipts. A pending tool announcement +is not a start. A completed tool status does not imply that a start was observed. A failed +status can reflect denial or failure before execution, so it does not establish +completed execution. + +All failed observations remain `observation_incomplete: true`. False means +**not observed**, not **did not happen**. Completion does not establish success, +a committed external effect, rollback, or replay safety. No automatic retry or +session reconstruction decision can be made from these facts alone. + +Fatal records use schema version 3 with an additive `possible_effects` object. +Version 1/2 records remain readable; a missing object means unknown. Existing +root-session failure paths without an explicit observation owner also report +unknown, rather than claim complete coverage. Child failures record the child +prompt snapshot in a parent-session-scoped diagnostic with surface `subagent`. + +## Current transport limitation + +The pinned AgentKit `ToolError` exposes `ExecutionFailed(String)` and unit +`Cancelled`. The existing parent error behavior is unchanged. Effects remain +in a typed child error and the local fatal diagnostic; they are **not yet** +transported to the parent as native structured failure metadata. A rendered +JSON string is not a substitute for the shared typed error contract. + +Do not return a completed `ToolResult` with `is_error: true` as a workaround: +the pinned Compose dispatcher treats completed child results as successful +values, bypassing Runlet error boundaries. Full #48 support still needs an +upstream typed error-metadata contract, including cancellation, plus root-session +and locally verified execution observation. Recovery (#21) must not infer that +an external harness can resume or fork an uncertain session from this metadata. diff --git a/src/acp_child.rs b/src/acp_child.rs index cd30eeda..563375a0 100644 --- a/src/acp_child.rs +++ b/src/acp_child.rs @@ -394,23 +394,56 @@ impl ChildConfig { } } -#[derive(Debug)] +#[derive(Clone, Debug)] pub(crate) enum ChildError { Cancelled, Failed(String), TerminalCancelled, TerminalFailed(String), + Observed { + error: Box, + effects: crate::effects::PossibleEffects, + }, } impl std::fmt::Display for ChildError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Cancelled | Self::TerminalCancelled => f.write_str("nested agent cancelled"), Self::Failed(e) | Self::TerminalFailed(e) => f.write_str(e), + Self::Observed { error, .. } => error.fmt(f), + } + } +} + +impl ChildError { + pub(crate) fn root(&self) -> &Self { + match self { + Self::Observed { error, .. } => error.root(), + error => error, + } + } + + pub(crate) fn possible_effects(&self) -> crate::effects::PossibleEffects { + match self { + Self::Observed { effects, .. } => *effects, + _ => crate::effects::PossibleEffects::default(), + } + } + + fn observed(self, observations: &crate::effects::Observations) -> Self { + self.with_effects(observations.snapshot()) + } + + pub(crate) fn with_effects(self, effects: crate::effects::PossibleEffects) -> Self { + Self::Observed { + error: Box::new(self), + effects, } } } struct Prompt { + observations: crate::effects::Observations, session_id: SessionId, text: String, cancellation: TurnCancellation, @@ -440,6 +473,7 @@ struct Ready { #[derive(Clone, Debug, Default)] pub(crate) struct ChildOutput { + observations: crate::effects::Observations, pub text: String, pub updates: Vec, pub updates_truncated: bool, @@ -447,7 +481,12 @@ pub(crate) struct ChildOutput { } impl ChildOutput { + pub(crate) fn possible_effects(&self) -> crate::effects::PossibleEffects { + self.observations.snapshot() + } + fn record(&mut self, update: SessionUpdate) { + self.observations.record(&update); if let SessionUpdate::AgentMessageChunk(chunk) = &update && let ContentBlock::Text(text) = &chunk.content { @@ -728,24 +767,30 @@ impl ChildSession { text: String, cancellation: TurnCancellation, ) -> Result { - let _serial = tokio::select! { - serial = self.serial.lock() => serial, - () = cancellation.cancelled() => return Err(ChildError::Cancelled), - }; - let (reply, response) = oneshot::channel(); - let request = Request::Prompt(Prompt { - session_id: self.session_id.clone(), - text, - cancellation: cancellation.clone(), - reply, - }); - tokio::select! { - sent = self.tx.send(request) => sent.map_err(|_| ChildError::TerminalFailed("nested agent process is no longer running".into()))?, - () = cancellation.cancelled() => return Err(ChildError::Cancelled), - } - response.await.map_err(|_| { - ChildError::TerminalFailed("nested agent process exited without a response".into()) - })? + // This owner outlives the prompt task, including channel loss/abort. + let observations = crate::effects::Observations::default(); + let outcome = async { + let _serial = tokio::select! { + serial = self.serial.lock() => serial, + () = cancellation.cancelled() => return Err(ChildError::Cancelled), + }; + let (reply, response) = oneshot::channel(); + let request = Request::Prompt(Prompt { + observations: observations.clone(), + session_id: self.session_id.clone(), + text, + cancellation: cancellation.clone(), + reply, + }); + tokio::select! { + sent = self.tx.send(request) => sent.map_err(|_| ChildError::TerminalFailed("nested agent process is no longer running".into()))?, + () = cancellation.cancelled() => return Err(ChildError::Cancelled), + } + response.await.map_err(|_| { + ChildError::TerminalFailed("nested agent process exited without a response".into()) + })? + }.await; + outcome.map_err(|error| error.observed(&observations)) } } @@ -1051,7 +1096,10 @@ async fn run( let fatal = fatal_tx.clone(); tasks.spawn(async move { let session_id = prompt.session_id.clone(); - let output = Arc::new(Mutex::new(ChildOutput::default())); + let output = Arc::new(Mutex::new(ChildOutput { + observations: prompt.observations.clone(), + ..ChildOutput::default() + })); if let Ok(mut routes) = routes.lock() { routes.insert(session_id.clone(), Arc::clone(&output)); } let request = connection.send_request(agentkit_acp::PromptRequest::new( session_id.clone(), vec![ContentBlock::Text(agentkit_acp::TextContent::new(prompt.text))], @@ -1231,6 +1279,63 @@ mod tests { use super::*; + #[test] + fn effect_observations_survive_output_retention_limits() { + let mut output = ChildOutput { + updates: vec![Value::Null; MAX_CAPTURED_UPDATES], + ..ChildOutput::default() + }; + output.record(update(json!({ + "sessionUpdate": "tool_call", "toolCallId": "secret-id", + "title": "secret command", "status": "in_progress" + }))); + assert!(output.updates_truncated); + assert!(output.observations.snapshot().tool_execution_start_reported); + output.record(update(json!({ + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": "private assistant output"} + }))); + let failure = ChildError::TerminalCancelled.observed(&output.observations); + assert!(failure.possible_effects().assistant_output_observed); + assert!(matches!(failure.root(), ChildError::TerminalCancelled)); + assert!(failure.possible_effects().observation_incomplete); + } + + #[tokio::test] + async fn reply_channel_loss_preserves_positive_observations() { + let (tx, mut rx) = mpsc::channel(1); + let (_closed, closed) = watch::channel(false); + let child = ChildSession { + tx, + session_id: "test".into(), + capabilities: agentkit_acp::AgentCapabilities::default(), + serial: Arc::new(tokio::sync::Mutex::new(())), + closed, + descendant_parent: None, + }; + let actor = tokio::spawn(async move { + let Some(Request::Prompt(prompt)) = rx.recv().await else { + panic!("prompt expected") + }; + prompt.observations.record(&update(json!({ + "sessionUpdate": "tool_call", "toolCallId": "t", + "title": "tool", "status": "completed" + }))); + // Simulate transport/task loss before the response is delivered. + drop(prompt.reply); + }); + let error = child + .prompt("private prompt".into(), TurnCancellation::default()) + .await + .unwrap_err(); + actor.await.unwrap(); + assert!(matches!(error.root(), ChildError::TerminalFailed(_))); + let effects = error.possible_effects(); + assert!(effects.tool_execution_completion_reported); + assert!(!effects.tool_execution_start_reported); + assert!(effects.observation_incomplete); + } + fn update(value: Value) -> SessionUpdate { serde_json::from_value(value).unwrap() } diff --git a/src/effects.rs b/src/effects.rs new file mode 100644 index 00000000..2e791446 --- /dev/null +++ b/src/effects.rs @@ -0,0 +1,154 @@ +//! Bounded positive observations, never a replay-safety decision. +//! ACP lifecycle statuses are reports from the harness, not execution receipts. + +use std::sync::{Arc, Mutex}; + +use agentkit_acp::{SessionUpdate, ToolCallStatus}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ObservationSource { + #[default] + Unknown, + AcpNotifications, +} + +/// False means only "not observed". Completion does not imply success, +/// rollback, a committed effect, or permission to repeat an operation. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct PossibleEffects { + pub source: ObservationSource, + pub assistant_output_observed: bool, + pub tool_emission_observed: bool, + pub tool_execution_start_reported: bool, + pub tool_execution_completion_reported: bool, + pub observation_incomplete: bool, +} + +impl Default for PossibleEffects { + fn default() -> Self { + Self { + source: ObservationSource::Unknown, + assistant_output_observed: false, + tool_emission_observed: false, + tool_execution_start_reported: false, + tool_execution_completion_reported: false, + // No failure path proves complete observation of external activity. + observation_incomplete: true, + } + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct Observations(Arc>); + +impl Observations { + pub(crate) fn snapshot(&self) -> PossibleEffects { + self.0.lock().map(|value| *value).unwrap_or_default() + } + + pub(crate) fn record(&self, update: &SessionUpdate) { + let Ok(mut effects) = self.0.lock() else { + return; + }; + effects.source = ObservationSource::AcpNotifications; + let status = match update { + SessionUpdate::AgentMessageChunk(_) => { + effects.assistant_output_observed = true; + None + } + SessionUpdate::ToolCall(call) => { + effects.tool_emission_observed = true; + Some(call.status) + } + SessionUpdate::ToolCallUpdate(update) => { + effects.tool_emission_observed = true; + update.fields.status + } + _ => None, + }; + match status { + Some(ToolCallStatus::InProgress) => effects.tool_execution_start_reported = true, + Some(ToolCallStatus::Completed) => { + // Do not manufacture a start observation from completion. + // Failed can mean pre-execution denial, not completed execution. + effects.tool_execution_completion_reported = true; + } + _ => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn update(value: serde_json::Value) -> SessionUpdate { + serde_json::from_value(value).unwrap() + } + + #[test] + fn observations_are_monotonic_and_do_not_infer_execution() { + let observations = Observations::default(); + assert_eq!(observations.snapshot(), PossibleEffects::default()); + observations.record(&update(json!({ + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": "secret output"} + }))); + assert!(observations.snapshot().assistant_output_observed); + observations.record(&update(json!({ + "sessionUpdate": "tool_call", "toolCallId": "private-id", + "title": "private command", "status": "pending", + "rawInput": {"secret": "private arguments"} + }))); + let emitted = observations.snapshot(); + assert!(emitted.tool_emission_observed); + assert!(!emitted.tool_execution_start_reported); + observations.record(&update(json!({ + "sessionUpdate": "tool_call_update", "toolCallId": "private-id", + "status": "completed", "rawOutput": "private result" + }))); + let completed = observations.snapshot(); + assert!(completed.tool_execution_completion_reported); + assert!(!completed.tool_execution_start_reported); + observations.record(&update(json!({ + "sessionUpdate": "tool_call_update", "toolCallId": "other", + "status": "in_progress" + }))); + let effects = observations.snapshot(); + assert!(effects.tool_execution_start_reported); + assert!(effects.tool_execution_completion_reported); + assert!(effects.observation_incomplete); + let envelope = serde_json::to_string(&effects).unwrap(); + assert!(!envelope.contains("secret")); + assert!(!envelope.contains("private")); + assert!(envelope.len() < 1024); + } + + #[test] + fn failed_status_does_not_prove_execution_completed() { + let observations = Observations::default(); + observations.record(&update(json!({ + "sessionUpdate": "tool_call_update", "toolCallId": "denied", + "status": "failed" + }))); + let effects = observations.snapshot(); + assert!(effects.tool_emission_observed); + assert!(!effects.tool_execution_start_reported); + assert!(!effects.tool_execution_completion_reported); + assert!(effects.observation_incomplete); + } + + #[test] + fn malformed_or_payload_bearing_metadata_is_rejected() { + let mut value = serde_json::to_value(PossibleEffects::default()).unwrap(); + value["tool_arguments"] = json!("secret"); + assert!(serde_json::from_value::(value).is_err()); + let mut value = serde_json::to_value(PossibleEffects::default()).unwrap(); + value["assistant_output_observed"] = json!("false"); + assert!(serde_json::from_value::(value).is_err()); + } +} diff --git a/src/fatal.rs b/src/fatal.rs index dd0e5e37..40d66a15 100644 --- a/src/fatal.rs +++ b/src/fatal.rs @@ -9,7 +9,7 @@ use agentkit_loop::LoopError; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use serde::{Deserialize, Serialize}; -const SCHEMA_VERSION: u64 = 2; +const SCHEMA_VERSION: u64 = 3; const MAX_MESSAGE_BYTES: usize = 2 * 1024; const MAX_RECORD_BYTES: usize = 32 * 1024; const MAX_RECORDS_PER_SESSION: usize = 50; @@ -25,6 +25,7 @@ pub(crate) enum Surface { A2a, Acp, Prompt, + Subagent, } impl Surface { @@ -33,6 +34,7 @@ impl Surface { Self::A2a => "a2a", Self::Acp => "acp", Self::Prompt => "prompt", + Self::Subagent => "subagent", } } } @@ -50,6 +52,8 @@ struct FatalRecord { message: String, #[serde(default, skip_serializing_if = "Option::is_none")] diagnostics: Option, + #[serde(default)] + possible_effects: crate::effects::PossibleEffects, } #[derive(Clone, Copy, Debug, Deserialize, Serialize)] @@ -436,6 +440,48 @@ fn write_in_with_diagnostics( code: &str, message: &str, diagnostics: Option<&TransportDiagnostics>, +) -> Result { + write_in_with_effects( + base, + session_id, + surface, + kind, + code, + message, + diagnostics, + crate::effects::PossibleEffects::default(), + ) +} + +pub(crate) fn record_child_failure( + session_id: &str, + effects: crate::effects::PossibleEffects, +) -> Result { + let home = std::env::var_os("HOME") + .filter(|home| !home.is_empty()) + .ok_or_else(|| "HOME is unset; cannot store fatal error log".to_owned())?; + write_in_with_effects( + &PathBuf::from(home).join(".kit/errors"), + session_id, + Surface::Subagent, + "runtime", + "subagent_failed", + "nested agent failed; effects and completion are unconfirmed", + None, + effects, + ) +} + +#[allow(clippy::too_many_arguments)] +fn write_in_with_effects( + base: &Path, + session_id: &str, + surface: Surface, + kind: &str, + code: &str, + message: &str, + diagnostics: Option<&TransportDiagnostics>, + possible_effects: crate::effects::PossibleEffects, ) -> Result { crate::session::validate_id(session_id)?; let occurred_at_ms = SystemTime::now() @@ -458,6 +504,7 @@ fn write_in_with_diagnostics( code: canonical_code(code).into(), message: bounded(message), diagnostics: diagnostics.filter(|value| value.valid()).cloned(), + possible_effects, }; let mut bytes = serde_json::to_vec_pretty(&record) .map_err(|error| format!("could not encode fatal error log: {error}"))?; @@ -614,12 +661,48 @@ mod tests { .unwrap(); assert_eq!(path.parent().unwrap(), root.path().join("session-1")); let record: FatalRecord = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - assert_eq!(record.schema_version, 2); + assert_eq!(record.schema_version, 3); assert_eq!(record.session_id, "session-1"); assert_eq!(record.surface, "prompt"); assert_eq!(record.code, "stream_transport"); } + #[test] + fn writes_effects_without_sensitive_content_and_reads_legacy_unknown() { + let root = tempfile::tempdir().unwrap(); + let effects = crate::effects::PossibleEffects { + source: crate::effects::ObservationSource::AcpNotifications, + assistant_output_observed: true, + ..crate::effects::PossibleEffects::default() + }; + let path = super::write_in_with_effects( + root.path(), + "session-effects", + Surface::Subagent, + "runtime", + "subagent_failed", + "nested agent failed", + None, + effects, + ) + .unwrap(); + let mut value: serde_json::Value = + serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let current: FatalRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(current.possible_effects, effects); + value["schema_version"] = json!(2); + value.as_object_mut().unwrap().remove("possible_effects"); + let legacy: FatalRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + legacy.possible_effects, + crate::effects::PossibleEffects::default() + ); + // An additive field on an older version must not be overwritten. + value["possible_effects"] = serde_json::to_value(effects).unwrap(); + let mixed: FatalRecord = serde_json::from_value(value).unwrap(); + assert_eq!(mixed.possible_effects, effects); + } + #[test] fn schema_v1_records_remain_readable() { let record: FatalRecord = serde_json::from_value(json!({ @@ -656,7 +739,7 @@ mod tests { let encoded = fs::read_to_string(path).unwrap(); let value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); - assert_eq!(value["schema_version"], 2); + assert_eq!(value["schema_version"], 3); assert_eq!(value["diagnostics"]["response_request_id"], "req_safe-123"); assert_eq!(value["diagnostics"]["stage"], "stream"); assert_eq!(value["diagnostics"]["retryable"], true); diff --git a/src/lib.rs b/src/lib.rs index b800e72e..df2dd21a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ mod compose_output; pub mod config_files; mod credentials; pub mod docs; +mod effects; pub mod events; mod fatal; mod file_search; diff --git a/src/tools/subagent.rs b/src/tools/subagent.rs index 4872986c..2b56736b 100644 --- a/src/tools/subagent.rs +++ b/src/tools/subagent.rs @@ -68,7 +68,8 @@ fn reserve_name(used: &mut HashSet, name: &str) -> bool { } fn child_error_is_terminal(error: &ChildError, child: &ChildSession) -> bool { - match error { + match error.root() { + ChildError::Observed { .. } => unreachable!("root unwraps observations"), ChildError::TerminalCancelled | ChildError::TerminalFailed(_) => true, ChildError::Cancelled | ChildError::Failed(_) => child.is_closed(), } @@ -237,6 +238,7 @@ struct CreateOptions { } struct ForkSuccess { + effects: crate::effects::PossibleEffects, value: SubagentValue, acknowledge: oneshot::Sender<()>, } @@ -440,9 +442,11 @@ impl Subagents { return Err(error); } }; + let effects = output.possible_effects(); let (output, updates) = turn_output(output, contract); let mut locked = state.lock().await; - self.check_active(&locked)?; + self.check_active(&locked) + .map_err(|error| error.with_effects(effects))?; locked.status = SubagentStatus::Idle; locked.outcome = Some(GenerationOutcome::Success); locked.generation_finished_at_unix_ms = Some(events::now_millis()); @@ -503,9 +507,11 @@ impl Subagents { .await { Ok(output) => { + let effects = output.possible_effects(); let (output, updates) = turn_output(output, contract); let mut locked = state.lock().await; - self.check_active(&locked)?; + self.check_active(&locked) + .map_err(|error| error.with_effects(effects))?; locked.status = SubagentStatus::Idle; locked.handle_generation = generation; locked.outcome = Some(GenerationOutcome::Success); @@ -614,7 +620,7 @@ impl Subagents { let result = manager.run_fork(operation, &reply).await; manager.finish_forking(&source_state, &reservation).await; match result { - Ok(value) => manager.handoff_fork_success(reply, value).await, + Ok((value, effects)) => manager.handoff_fork_success(reply, value, effects).await, Err(error) => { let _ = reply.send(Err(error)); } @@ -628,6 +634,7 @@ impl Subagents { ChildError::Failed( "subagent fork task stopped before transferring ownership".into(), ) + .with_effects(success.effects) })?; Ok(success.value) } @@ -639,7 +646,7 @@ impl Subagents { &self, operation: ForkOperation, reply: &oneshot::Sender>, - ) -> Result { + ) -> Result<(SubagentValue, crate::effects::PossibleEffects), ChildError> { let ForkOperation { source_id, source_state, @@ -789,18 +796,24 @@ impl Subagents { .await); } }; + let effects = output.possible_effects(); let (output, updates) = turn_output(output, contract.as_deref()); let mut locked = state.lock().await; if reply.is_closed() { drop(locked); return Err(self - .cleanup_installed_child(&id, &state, &child, ChildError::Cancelled) + .cleanup_installed_child( + &id, + &state, + &child, + ChildError::Cancelled.with_effects(effects), + ) .await); } if let Err(error) = self.check_active(&locked) { drop(locked); return Err(self - .cleanup_installed_child(&id, &state, &child, error) + .cleanup_installed_child(&id, &state, &child, error.with_effects(effects)) .await); } locked.status = SubagentStatus::Idle; @@ -812,13 +825,16 @@ impl Subagents { let event = locked.runtime_event(id.clone()); drop(locked); self.emit_event(event); - Ok(SubagentValue { - id, - name: Some(name), - output, - generation, - updates, - }) + Ok(( + SubagentValue { + id, + name: Some(name), + output, + generation, + updates, + }, + effects, + )) } async fn list( @@ -973,10 +989,17 @@ impl Subagents { &self, reply: oneshot::Sender>, value: SubagentValue, + effects: crate::effects::PossibleEffects, ) { let cleanup = value.clone(); let (acknowledge, acknowledged) = oneshot::channel(); - let sent = reply.send(Ok(ForkSuccess { value, acknowledge })).is_ok(); + let sent = reply + .send(Ok(ForkSuccess { + value, + effects, + acknowledge, + })) + .is_ok(); if !sent || acknowledged.await.is_err() { self.cleanup_abandoned_fork(&cleanup).await; } @@ -1035,11 +1058,10 @@ impl Subagents { match child.close().await { Ok(()) => error, Err(cleanup) if child_error_is_terminal(&cleanup, &child) => error, - Err(cleanup) => { + Err(_) => { Self::watch_permit_until_process_exit(permit, &child); - ChildError::Failed(format!( - "{error}; failed to clean up retired subagent session: {cleanup}" - )) + tracing::warn!("failed to clean up retired subagent session"); + error } } } @@ -1070,11 +1092,10 @@ impl Subagents { match child.close().await { Ok(()) => error, Err(cleanup) if child_error_is_terminal(&cleanup, child) => error, - Err(cleanup) => { + Err(_) => { self.retain_permit_until_process_exit(state, child).await; - ChildError::Failed(format!( - "{error}; failed to clean up retired subagent session: {cleanup}" - )) + tracing::warn!("failed to clean up retired subagent session"); + error } } } @@ -1087,6 +1108,7 @@ impl Subagents { error: ChildError, ) -> ChildError { let manager = self.clone(); + let fallback = error.clone(); match tokio::spawn(async move { manager .cleanup_installed_child(&id, &state, &child, error) @@ -1095,8 +1117,9 @@ impl Subagents { .await { Ok(error) => error, - Err(error) => { - ChildError::Failed(format!("retired subagent cleanup task failed: {error}")) + Err(_) => { + tracing::warn!("retired subagent cleanup task failed"); + fallback } } } @@ -1459,15 +1482,26 @@ fn cancellation(context: &ToolContext<'_>) -> TurnCancellation { .map(|value| value.handle().checkpoint()) .unwrap_or_default() } +fn tool_failure(error: &ChildError) -> ToolError { + match error.root() { + ChildError::Cancelled | ChildError::TerminalCancelled => ToolError::Cancelled, + ChildError::Failed(message) | ChildError::TerminalFailed(message) => { + ToolError::ExecutionFailed(message.clone()) + } + ChildError::Observed { .. } => unreachable!("root unwraps observations"), + } +} + fn result( request: ToolRequest, value: Result, ) -> Result { - let value = value.map_err(|error| match error { - ChildError::Cancelled | ChildError::TerminalCancelled => ToolError::Cancelled, - ChildError::Failed(error) | ChildError::TerminalFailed(error) => { - ToolError::ExecutionFailed(error) + let value = value.map_err(|error| { + let effects = error.possible_effects(); + if let Err(log_error) = crate::fatal::record_child_failure(&request.session_id.0, effects) { + tracing::warn!(%log_error, "could not store child failure observations"); } + tool_failure(&error) })?; Ok(ToolResult::new(ToolResultPart::success( request.call_id, diff --git a/src/tools/subagent/tests.rs b/src/tools/subagent/tests.rs index d3a62a87..da44321e 100644 --- a/src/tools/subagent/tests.rs +++ b/src/tools/subagent/tests.rs @@ -1472,7 +1472,9 @@ async fn successful_fork_handoff_cleans_up_if_receipt_is_not_acknowledged() { let manager = scenario.manager.clone(); let cleanup_branch = branch.clone(); let handoff = tokio::spawn(async move { - manager.handoff_fork_success(reply, cleanup_branch).await; + manager + .handoff_fork_success(reply, cleanup_branch, Default::default()) + .await; }); let success = response.await.unwrap().unwrap(); @@ -1855,3 +1857,130 @@ async fn generic_harness_without_native_fork_returns_unsupported() { "ACP harness \"acp.generic\" does not advertise session/fork; transcript fallback is only available for Kit" ); } + +#[tokio::test] +async fn observed_failures_keep_terminal_and_cancellation_classification() { + let (child, _) = ChildSession::closure_probe_for_test(); + let effects = crate::effects::PossibleEffects { + assistant_output_observed: true, + ..crate::effects::PossibleEffects::default() + }; + for error in [ + ChildError::TerminalCancelled, + ChildError::TerminalFailed("transport ended".into()), + ] { + let observed = ChildError::Observed { + error: Box::new(error), + effects, + }; + assert!(child_error_is_terminal(&observed, &child)); + assert_eq!(observed.possible_effects(), effects); + match observed.root() { + ChildError::TerminalCancelled => { + assert!(matches!(tool_failure(&observed), ToolError::Cancelled)) + } + _ => assert!( + matches!(tool_failure(&observed), ToolError::ExecutionFailed(message) if message == "transport ended") + ), + } + } + let observed = ChildError::Observed { + error: Box::new(ChildError::Failed("refused".into())), + effects, + }; + assert!(!child_error_is_terminal(&observed, &child)); + assert!( + matches!(tool_failure(&observed), ToolError::ExecutionFailed(message) if message == "refused") + ); +} + +#[tokio::test] +async fn successful_prompt_after_retirement_keeps_observed_effects() { + let scenario = MockAcpScenario::new(ScenarioOptions { + gate_prompt: Some("MOCK_RICH_OUTPUT"), + ..Default::default() + }); + let source = scenario.create("source").await; + let prompt_manager = scenario.manager.clone(); + let prompt_source = source.clone(); + let prompt = tokio::spawn(async move { + prompt_manager + .prompt( + prompt_source, + "MOCK_RICH_OUTPUT".into(), + TurnCancellation::default(), + None, + ) + .await + }); + scenario + .wait_for(|request| { + matches!(request, LoggedRequest::Prompt { text, .. } if text == "MOCK_RICH_OUTPUT") + }) + .await; + scenario + .manager + .close(&source.id, &TurnCancellation::default()) + .await + .unwrap(); + MockAcpScenario::release(&scenario.prompt_release); + let error = prompt.await.unwrap().unwrap_err(); + let effects = error.possible_effects(); + assert!(effects.assistant_output_observed); + assert!(effects.tool_emission_observed); + assert!(effects.tool_execution_completion_reported); + assert!(effects.observation_incomplete); + assert!( + scenario + .manager + .list(&TurnCancellation::default()) + .await + .unwrap() + .is_empty() + ); +} + +#[tokio::test] +async fn failed_cleanup_preserves_cancellation_and_holds_capacity() { + for original in [ChildError::Cancelled, ChildError::TerminalCancelled] { + let scenario = MockAcpScenario::new(ScenarioOptions { + fail_close_session: Some("branch-1"), + ..Default::default() + }); + let source = scenario.create("source").await; + let branch = scenario + .spawn_fork(source.clone(), "branch") + .await + .unwrap() + .unwrap(); + let state = scenario.manager.lookup(&branch).unwrap(); + let child = state.lock().await.child.clone().unwrap(); + let effects = crate::effects::PossibleEffects { + assistant_output_observed: true, + ..Default::default() + }; + let terminal = matches!(original, ChildError::TerminalCancelled); + let error = scenario + .manager + .cleanup_installed_child(&branch.id, &state, &child, original.with_effects(effects)) + .await; + assert_eq!( + matches!(error.root(), ChildError::TerminalCancelled), + terminal + ); + assert!(matches!(tool_failure(&error), ToolError::Cancelled)); + assert_eq!(error.possible_effects(), effects); + assert_eq!( + scenario.manager.capacity.available_permits(), + MAX_LIVE_SUBAGENTS - 2 + ); + drop(state); + drop(child); + scenario + .manager + .close(&source.id, &TurnCancellation::default()) + .await + .unwrap(); + wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; + } +} From a4a903bc50a522a4c9c16217e2b5990ac6c46927 Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 20:02:27 +0100 Subject: [PATCH 2/7] fix(runtime): retain conservative possible-effects diagnostics --- docs/issues/possible-effects.md | 96 ++++--- src/effects.rs | 273 +++++++++++++++++++- src/fatal.rs | 105 ++++++-- src/protocols/a2a.rs | 85 +++++-- src/protocols/acp.rs | 431 ++++++++++++++++++++++++++++---- src/protocols/acp/activity.rs | 50 +++- src/protocols/acp/v2.rs | 330 +++++++++++++++++++++++- src/runtime.rs | 255 +++++++++++++++---- src/runtime/tests.rs | 87 +++++++ src/tools/observed.rs | 293 +++++++++++++++++++--- src/tools/subagent.rs | 80 +++--- src/tools/subagent/tests.rs | 265 ++++++++++++++++++-- tests/runtime.rs | 14 +- 13 files changed, 2082 insertions(+), 282 deletions(-) diff --git a/docs/issues/possible-effects.md b/docs/issues/possible-effects.md index 0ff9f1eb..fe409ad2 100644 --- a/docs/issues/possible-effects.md +++ b/docs/issues/possible-effects.md @@ -1,41 +1,71 @@ # Possible-effects observations (issue #48) -A failed child prompt can have changed external state. Kit retains bounded, -provider-independent positive observations from ACP notifications, independently -of its output-retention limits and opt-in stderr display events. The owner of the -observations outlives the dispatched prompt task, so reply-channel loss does not -silently discard previously observed activity. +A failed prompt can have changed external state. Kit retains bounded, +provider-independent positive observations independently of output-retention +limits and opt-in stderr display events. `possible_effects` contains only fixed field names, an allowlisted source, and booleans. It never contains tool IDs, arguments, results, assistant text, prompts, -or provider payloads. `source: acp_notifications` identifies lifecycle **reports** -from a harness, not verified local execution receipts. A pending tool announcement -is not a start. A completed tool status does not imply that a start was observed. A failed -status can reflect denial or failure before execution, so it does not establish -completed execution. - -All failed observations remain `observation_incomplete: true`. False means -**not observed**, not **did not happen**. Completion does not establish success, -a committed external effect, rollback, or replay safety. No automatic retry or -session reconstruction decision can be made from these facts alone. - -Fatal records use schema version 3 with an additive `possible_effects` object. -Version 1/2 records remain readable; a missing object means unknown. Existing -root-session failure paths without an explicit observation owner also report -unknown, rather than claim complete coverage. Child failures record the child -prompt snapshot in a parent-session-scoped diagnostic with surface `subagent`. - -## Current transport limitation - -The pinned AgentKit `ToolError` exposes `ExecutionFailed(String)` and unit -`Cancelled`. The existing parent error behavior is unchanged. Effects remain -in a typed child error and the local fatal diagnostic; they are **not yet** -transported to the parent as native structured failure metadata. A rendered -JSON string is not a substitute for the shared typed error contract. +or provider payloads. Sources distinguish two observation scopes: + +- `acp_notifications`: reports received during one dispatched child prompt. + Pending tool announcements are not starts. A completed status does not imply + that a start was observed. A failed status can reflect pre-execution denial, + so it does not establish completed execution. +- `local_session`: cumulative observations during one **live root owner's + lifetime**, shared by its loop observer, compose, and hidden tool wrappers. + These are not facts attributable exclusively to the failing prompt, nor do + they cover earlier process instances or a resumed session's stored history. + Execution starts and completions come from actual invocation entry/terminal + return boundaries, not synthesized tool-result events. Entry into a local + invocation does not establish dispatch or execution of an external operation. +- `unknown`: no explicit observation source is available. + +Local assistant-output tracking excludes reasoning and tool-argument deltas, +handles committed content without prior deltas, and bounds transient part +classification by identifier length and entry count. Presentation supersession +and new prompts clear only transient classification, never positive evidence. +Detached work retains the same owner when it spans prompts. + +All failure snapshots remain `observation_incomplete: true`. False means +**not observed**, not **did not happen**. A completion boolean means some +invocation completed; it does not establish that every invocation or external +operation completed. Interruptions and dropped futures are not completions. +Completion establishes neither successful external effects, rollback, nor replay +safety. No automatic retry or reconstruction decision follows from these facts. + +Child observation ownership survives dispatched-task/channel loss and +post-response ownership rejection. Cleanup failures preserve the original +failure classification. Root finalization may return a cleanup or flush error instead +of the original driver error; this gets a separate `session_finalization` +diagnostic without replacing the earlier driver record. Root prompt, ACP v1/v2, and A2A failure paths retain +explicit owners, including unstructured and unsolicited execution. Cancellation +keeps its existing lifecycle classification and records a snapshot after +structured cleanup when available; Kit does not wait for unrelated detached work +merely to improve evidence. + +Fatal writers use schema version 4, adding the `local_session` source to version +3's effects object. New readers retain v1/v2 missing-effects defaults and v3 +`unknown`/`acp_notifications` records; existing supplied effects are preserved. +Malformed fields, false completeness claims, and unknown future source values are rejected. This backward +read compatibility does not mean an older strict reader understands a new +source value. Files are not rewritten on read. Parent-side observation records +are distinct diagnostics, not recreated child fatal receipt identities. + +## Remaining typed transport and release gates + +The pinned AgentKit `ToolError` exposes string execution failures and unit +cancellation. The existing parent error behavior remains unchanged. The shared +upstream API implementation is authorized and reliability-owned; Kit integration +waits for reviewed compatible published crates rather than Cargo patches. + +Required integration is typed, validated metadata through ACP error data, child +failures, subagent tools, Runlet's documented catch/rethrow policy, and both +foreground/background terminal projections, retaining one child fatal receipt +with explicit storage disposition. Cancellation must keep its classification. +A rendered JSON string is not a substitute for that contract. Do not return a completed `ToolResult` with `is_error: true` as a workaround: the pinned Compose dispatcher treats completed child results as successful -values, bypassing Runlet error boundaries. Full #48 support still needs an -upstream typed error-metadata contract, including cancellation, plus root-session -and locally verified execution observation. Recovery (#21) must not infer that -an external harness can resume or fork an uncertain session from this metadata. +values, bypassing Runlet error boundaries. Recovery (#21) remains gated until +complete #48 delivery and review; effects alone cannot reconstruct a session. diff --git a/src/effects.rs b/src/effects.rs index 2e791446..f6088b1f 100644 --- a/src/effects.rs +++ b/src/effects.rs @@ -1,7 +1,13 @@ //! Bounded positive observations, never a replay-safety decision. //! ACP lifecycle statuses are reports from the harness, not execution receipts. -use std::sync::{Arc, Mutex}; +use std::{ + collections::HashSet, + sync::{Arc, Mutex}, +}; + +use agentkit_core::{Delta, Part, PartId, PartKind}; +use agentkit_loop::{AgentEvent, LoopObserver, ObservedEvent}; use agentkit_acp::{SessionUpdate, ToolCallStatus}; use serde::{Deserialize, Serialize}; @@ -12,6 +18,8 @@ pub(crate) enum ObservationSource { #[default] Unknown, AcpNotifications, + /// Cumulative observations during this live root owner's lifetime only. + LocalSession, } /// False means only "not observed". Completion does not imply success, @@ -24,9 +32,22 @@ pub(crate) struct PossibleEffects { pub tool_emission_observed: bool, pub tool_execution_start_reported: bool, pub tool_execution_completion_reported: bool, + #[serde(deserialize_with = "incomplete_observation")] pub observation_incomplete: bool, } +fn incomplete_observation<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result { + if bool::deserialize(deserializer)? { + Ok(true) + } else { + Err(serde::de::Error::custom( + "failure observation must remain incomplete", + )) + } +} + impl Default for PossibleEffects { fn default() -> Self { Self { @@ -41,18 +62,115 @@ impl Default for PossibleEffects { } } +const MAX_TRACKED_PARTS: usize = 128; +const MAX_PART_ID_BYTES: usize = 256; + +#[derive(Debug, Default)] +struct ObservationState { + effects: PossibleEffects, + // Transient classification only; never serialized into diagnostics. + assistant_parts: HashSet, +} + #[derive(Clone, Debug, Default)] -pub(crate) struct Observations(Arc>); +pub(crate) struct Observations(Arc>); impl Observations { + pub(crate) fn local_session() -> Self { + Self(Arc::new(Mutex::new(ObservationState { + effects: PossibleEffects { + source: ObservationSource::LocalSession, + ..Default::default() + }, + ..Default::default() + }))) + } + + pub(crate) fn invocation_started(&self) { + if let Ok(mut state) = self.0.lock() { + state.effects.tool_execution_start_reported = true; + } + } + + pub(crate) fn invocation_completed(&self) { + if let Ok(mut state) = self.0.lock() { + state.effects.tool_execution_completion_reported = true; + } + } + + fn observe_local(&self, event: &AgentEvent) { + let Ok(mut state) = self.0.lock() else { return }; + match event { + AgentEvent::ToolCallRequested(_) => state.effects.tool_emission_observed = true, + AgentEvent::ContentDelta(delta) => match delta { + Delta::BeginPart { part_id, kind } => { + state.assistant_parts.remove(part_id); + if *kind == PartKind::ToolCall { + state.effects.tool_emission_observed = true; + } + if matches!( + kind, + PartKind::Text | PartKind::Media | PartKind::File | PartKind::Structured + ) && part_id.0.len() <= MAX_PART_ID_BYTES + && state.assistant_parts.len() < MAX_TRACKED_PARTS + { + state.assistant_parts.insert(part_id.clone()); + } + } + Delta::AppendText { part_id, chunk } => { + if !chunk.is_empty() && state.assistant_parts.contains(part_id) { + state.effects.assistant_output_observed = true; + } + } + Delta::AppendBytes { part_id, chunk } => { + if !chunk.is_empty() && state.assistant_parts.contains(part_id) { + state.effects.assistant_output_observed = true; + } + } + Delta::ReplaceStructured { part_id, .. } => { + if state.assistant_parts.contains(part_id) { + state.effects.assistant_output_observed = true; + } + } + Delta::CommitPart { part } => { + match part { + Part::Text(text) if !text.text.is_empty() => { + state.effects.assistant_output_observed = true + } + Part::Media(_) | Part::File(_) | Part::Structured(_) => { + state.effects.assistant_output_observed = true + } + Part::ToolCall(_) => state.effects.tool_emission_observed = true, + _ => {} + } + // CommitPart carries no PartId. Forget classifications rather + // than risk applying a stale kind to a later reused id. + state.assistant_parts.clear(); + } + Delta::SetMetadata { .. } => {} + }, + AgentEvent::TurnStarted { .. } + | AgentEvent::TurnFinished(_) + | AgentEvent::ResponseAttemptSuperseded => { + // Presentation/turn boundaries cannot erase effects, particularly + // when a detached invocation spans more than one prompt. + state.assistant_parts.clear(); + } + // ToolResultReceived includes synthetic permission/cancellation + // results. Only actual invocation boundaries report execution. + _ => {} + } + } + pub(crate) fn snapshot(&self) -> PossibleEffects { - self.0.lock().map(|value| *value).unwrap_or_default() + self.0.lock().map(|value| value.effects).unwrap_or_default() } pub(crate) fn record(&self, update: &SessionUpdate) { - let Ok(mut effects) = self.0.lock() else { + let Ok(mut state) = self.0.lock() else { return; }; + let effects = &mut state.effects; effects.source = ObservationSource::AcpNotifications; let status = match update { SessionUpdate::AgentMessageChunk(_) => { @@ -81,6 +199,47 @@ impl Observations { } } +impl LoopObserver for Observations { + fn handle_event(&self, event: ObservedEvent) { + self.observe_local(&event.event); + } +} + +#[cfg(test)] +pub(crate) fn isolated_test(name: &str) -> bool { + if std::env::var("KIT_EFFECTS_TEST_CHILD").as_deref() == Ok(name) { + return false; + } + let home = tempfile::tempdir().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", name, "--nocapture"]) + .env("KIT_EFFECTS_TEST_CHILD", name) + .env("HOME", home.path()) + .env_remove(crate::events::EVENTS_ENV) + .output() + .unwrap(); + assert!( + output.status.success(), + "isolated effects test failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + true +} + +#[cfg(test)] +pub(crate) fn test_record(session_id: &str) -> serde_json::Value { + let directory = std::path::PathBuf::from(std::env::var_os("HOME").unwrap()) + .join(".kit/errors") + .join(session_id); + let mut files: Vec<_> = std::fs::read_dir(directory) + .unwrap() + .map(|file| file.unwrap().path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "json")) + .collect(); + files.sort(); + serde_json::from_slice(&std::fs::read(files.last().unwrap()).unwrap()).unwrap() +} + #[cfg(test)] mod tests { use super::*; @@ -144,6 +303,9 @@ mod tests { #[test] fn malformed_or_payload_bearing_metadata_is_rejected() { + let mut complete = serde_json::to_value(PossibleEffects::default()).unwrap(); + complete["observation_incomplete"] = json!(false); + assert!(serde_json::from_value::(complete).is_err()); let mut value = serde_json::to_value(PossibleEffects::default()).unwrap(); value["tool_arguments"] = json!("secret"); assert!(serde_json::from_value::(value).is_err()); @@ -151,4 +313,107 @@ mod tests { value["assistant_output_observed"] = json!("false"); assert!(serde_json::from_value::(value).is_err()); } + #[test] + fn local_content_classification_is_bounded_and_never_invents_output() { + let observations = Observations::local_session(); + let emit = |delta| observations.observe_local(&AgentEvent::ContentDelta(delta)); + let id = PartId::new("private-part"); + emit(Delta::BeginPart { + part_id: id.clone(), + kind: PartKind::Reasoning, + }); + emit(Delta::AppendText { + part_id: id.clone(), + chunk: "private reasoning".into(), + }); + emit(Delta::BeginPart { + part_id: id.clone(), + kind: PartKind::ToolCall, + }); + emit(Delta::AppendText { + part_id: id.clone(), + chunk: "private arguments".into(), + }); + assert!(!observations.snapshot().assistant_output_observed); + assert!(observations.snapshot().tool_emission_observed); + assert!(!observations.snapshot().tool_execution_start_reported); + emit(Delta::BeginPart { + part_id: id.clone(), + kind: PartKind::Text, + }); + assert!(!observations.snapshot().assistant_output_observed); + emit(Delta::AppendText { + part_id: id, + chunk: "private answer".into(), + }); + assert!(observations.snapshot().assistant_output_observed); + observations.observe_local(&AgentEvent::ResponseAttemptSuperseded); + observations.observe_local(&AgentEvent::TurnStarted { + session_id: "session".into(), + turn_id: "next".into(), + }); + assert!(observations.snapshot().assistant_output_observed); + let encoded = serde_json::to_string(&observations.snapshot()).unwrap(); + assert!(!encoded.contains("private")); + assert_eq!( + observations.snapshot().source, + ObservationSource::LocalSession + ); + assert!(observations.snapshot().observation_incomplete); + + let bounded = Observations::local_session(); + for index in 0..MAX_TRACKED_PARTS + 3 { + bounded.observe_local(&AgentEvent::ContentDelta(Delta::BeginPart { + part_id: format!("part-{index}").into(), + kind: PartKind::Text, + })); + } + assert_eq!( + bounded.0.lock().unwrap().assistant_parts.len(), + MAX_TRACKED_PARTS + ); + bounded.observe_local(&AgentEvent::ContentDelta(Delta::AppendText { + part_id: format!("part-{}", MAX_TRACKED_PARTS + 2).into(), + chunk: "unclassified".into(), + })); + let huge = PartId::new("x".repeat(MAX_PART_ID_BYTES + 1)); + bounded.observe_local(&AgentEvent::ContentDelta(Delta::BeginPart { + part_id: huge.clone(), + kind: PartKind::Text, + })); + bounded.observe_local(&AgentEvent::ContentDelta(Delta::AppendText { + part_id: huge, + chunk: "unclassified".into(), + })); + assert!(!bounded.snapshot().assistant_output_observed); + bounded.observe_local(&AgentEvent::ContentDelta(Delta::CommitPart { + part: Part::text("commit-only output"), + })); + assert!(bounded.snapshot().assistant_output_observed); + assert!(bounded.0.lock().unwrap().assistant_parts.is_empty()); + } + + #[test] + fn synthesized_results_are_not_invocation_receipts_and_sessions_are_isolated() { + let first = Observations::local_session(); + let second = Observations::local_session(); + first.observe_local(&AgentEvent::ToolResultReceived( + agentkit_core::ToolResultPart::error( + "denied", + agentkit_core::ToolOutput::text("denied"), + ), + )); + assert!(!first.snapshot().tool_execution_start_reported); + assert!(!first.snapshot().tool_execution_completion_reported); + let background = first.clone(); + background.invocation_started(); + first.observe_local(&AgentEvent::TurnStarted { + session_id: "session".into(), + turn_id: "next".into(), + }); + background.invocation_completed(); + assert!(first.snapshot().tool_execution_start_reported); + assert!(first.snapshot().tool_execution_completion_reported); + assert!(!second.snapshot().tool_execution_start_reported); + } } diff --git a/src/fatal.rs b/src/fatal.rs index 40d66a15..5585ffed 100644 --- a/src/fatal.rs +++ b/src/fatal.rs @@ -9,7 +9,7 @@ use agentkit_loop::LoopError; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use serde::{Deserialize, Serialize}; -const SCHEMA_VERSION: u64 = 3; +const SCHEMA_VERSION: u64 = 4; const MAX_MESSAGE_BYTES: usize = 2 * 1024; const MAX_RECORD_BYTES: usize = 32 * 1024; const MAX_RECORDS_PER_SESSION: usize = 50; @@ -230,37 +230,51 @@ fn split_diagnostics(message: &str) -> (&str, Option) { (plain, Some(diagnostics)) } -pub(crate) fn record_loop_error( +/// Explicit observation owner, including cancellation without reclassifying it. +pub(crate) fn record_loop_error_with_effects( session_id: &str, surface: Surface, error: &LoopError, + effects: crate::effects::PossibleEffects, ) -> Result, String> { - let Some((kind, code, message, diagnostics)) = classify(error) else { + let (kind, code, message, diagnostics) = if matches!(error, LoopError::Cancelled) { + ( + "cancelled", + "cancelled", + "execution cancelled; effects and completion are unconfirmed".into(), + None, + ) + } else if let Some(classified) = classify(error) { + classified + } else { return Ok(None); }; - write_default( + write_default_with_effects( session_id, surface, kind, code, &message, diagnostics.as_ref(), + effects, ) .map(Some) } -pub(crate) fn record_runtime_error( +pub(crate) fn record_runtime_error_with_effects( session_id: &str, surface: Surface, code: &str, + effects: crate::effects::PossibleEffects, ) -> Result { - write_default( + write_default_with_effects( session_id, surface, "runtime", canonical_code(code), "runtime failed before the session could continue", None, + effects, ) } @@ -398,18 +412,19 @@ fn canonical_code(code: &str) -> &str { } } -fn write_default( +fn write_default_with_effects( session_id: &str, surface: Surface, kind: &str, code: &str, message: &str, diagnostics: Option<&TransportDiagnostics>, + effects: crate::effects::PossibleEffects, ) -> Result { let home = std::env::var_os("HOME") .filter(|home| !home.is_empty()) .ok_or_else(|| "HOME is unset; cannot store fatal error log".to_owned())?; - write_in_with_diagnostics( + write_in_with_effects( &PathBuf::from(home).join(".kit/errors"), session_id, surface, @@ -417,6 +432,7 @@ fn write_default( code, message, diagnostics, + effects, ) } @@ -432,6 +448,7 @@ fn write_in( write_in_with_diagnostics(base, session_id, surface, kind, code, message, None) } +#[cfg(test)] fn write_in_with_diagnostics( base: &Path, session_id: &str, @@ -590,8 +607,8 @@ mod tests { use super::{ DIAGNOSTIC_MARKER, FatalRecord, H2Reason, IoClassification, MAX_DIAGNOSTIC_BYTES, MAX_RECORDS_PER_SESSION, ReqwestDiagnostics, Surface, TransportDiagnostics, - TransportSource, TransportStage, bounded, classify, event_order, record_loop_error, - render_loop_error, split_diagnostics, write_in, write_in_with_diagnostics, + TransportSource, TransportStage, bounded, classify, event_order, render_loop_error, + split_diagnostics, write_in, write_in_with_diagnostics, }; fn append_diagnostics(message: String, diagnostics: &TransportDiagnostics) -> String { @@ -661,7 +678,7 @@ mod tests { .unwrap(); assert_eq!(path.parent().unwrap(), root.path().join("session-1")); let record: FatalRecord = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - assert_eq!(record.schema_version, 3); + assert_eq!(record.schema_version, 4); assert_eq!(record.session_id, "session-1"); assert_eq!(record.surface, "prompt"); assert_eq!(record.code, "stream_transport"); @@ -739,7 +756,7 @@ mod tests { let encoded = fs::read_to_string(path).unwrap(); let value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); - assert_eq!(value["schema_version"], 3); + assert_eq!(value["schema_version"], 4); assert_eq!(value["diagnostics"]["response_request_id"], "req_safe-123"); assert_eq!(value["diagnostics"]["stage"], "stream"); assert_eq!(value["diagnostics"]["retryable"], true); @@ -867,9 +884,8 @@ mod tests { } #[test] - fn cancellation_is_not_recorded() { - let result = record_loop_error("session-1", Surface::Acp, &LoopError::Cancelled).unwrap(); - assert!(result.is_none()); + fn cancellation_is_distinct_from_provider_failure_classification() { + assert!(classify(&LoopError::Cancelled).is_none()); } #[test] @@ -946,4 +962,63 @@ mod tests { 0o700 ); } + #[test] + fn local_cancellation_has_conservative_post_cleanup_metadata() { + if crate::effects::isolated_test( + "fatal::tests::local_cancellation_has_conservative_post_cleanup_metadata", + ) { + return; + } + let observations = crate::effects::Observations::local_session(); + observations.invocation_started(); + let cleanup = observations.clone(); + cleanup.invocation_completed(); + let path = super::record_loop_error_with_effects( + "root-cancelled", + Surface::Acp, + &LoopError::Cancelled, + observations.snapshot(), + ) + .unwrap() + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!(value["schema_version"], 4); + assert_eq!(value["kind"], "cancelled"); + assert_eq!(value["code"], "cancelled"); + assert_eq!(value["possible_effects"]["source"], "local_session"); + assert_eq!( + value["possible_effects"]["tool_execution_completion_reported"], + true + ); + assert_eq!(value["possible_effects"]["observation_incomplete"], true); + } + + #[test] + fn schema_v3_effects_and_new_local_source_remain_strict() { + let root = tempfile::tempdir().unwrap(); + let path = write_in( + root.path(), + "session-schema", + Surface::Prompt, + "runtime", + "failed", + "failed", + ) + .unwrap(); + let mut value: serde_json::Value = + serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + value["schema_version"] = json!(3); + for source in ["unknown", "acp_notifications", "local_session"] { + value["possible_effects"]["source"] = json!(source); + let record: FatalRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + serde_json::to_value(record.possible_effects).unwrap()["source"], + source + ); + } + value["possible_effects"]["source"] = json!("unknown-future-source"); + assert!(serde_json::from_value::(value.clone()).is_err()); + value["possible_effects"] = json!({"source": "local_session", "private_payload": "secret"}); + assert!(serde_json::from_value::(value).is_err()); + } } diff --git a/src/protocols/a2a.rs b/src/protocols/a2a.rs index b690c56c..ca529362 100644 --- a/src/protocols/a2a.rs +++ b/src/protocols/a2a.rs @@ -18,6 +18,30 @@ use crate::runtime::Runtime; struct KitAgent(Arc); +fn record_failure( + session_id: &str, + error: &agentkit_loop::LoopError, + observations: &crate::effects::Observations, +) -> String { + let rendered = crate::fatal::render_loop_error(error); + match crate::fatal::record_loop_error_with_effects( + session_id, + crate::fatal::Surface::A2a, + error, + observations.snapshot(), + ) { + Ok(Some(path)) => eprintln!( + "stored fatal error log for {session_id}: {}", + path.display() + ), + Ok(None) => {} + Err(log_error) => { + eprintln!("could not store fatal error log for {session_id}: {log_error}") + } + } + rendered +} + impl AgentExecutor for KitAgent { fn execute<'a>( &'a self, @@ -45,9 +69,15 @@ impl AgentExecutor for KitAgent { emit.status(TaskState::Failed).await?; return Ok(()); } + let observations = crate::effects::Observations::local_session(); match self .0 - .run_cancelled(prompt, 0, Some(context.cancellation_token.clone())) + .run_cancelled_observed( + prompt, + 0, + Some(context.cancellation_token.clone()), + observations.clone(), + ) .await { Ok(output) => { @@ -57,27 +87,7 @@ impl AgentExecutor for KitAgent { } Err(error) => { let session_id = a2a_session_id(context); - let rendered = crate::fatal::render_loop_error(&error); - let rendered = match crate::fatal::record_loop_error( - &session_id, - crate::fatal::Surface::A2a, - &error, - ) { - Ok(Some(path)) => { - eprintln!( - "stored fatal error log for {session_id}: {}", - path.display() - ); - rendered - } - Ok(None) => rendered, - Err(log_error) => { - eprintln!( - "could not store fatal error log for {session_id}: {log_error}" - ); - rendered - } - }; + let rendered = record_failure(&session_id, &error, &observations); emit.artifact("error", vec![Part::text(rendered)], None, Some(true)) .await?; emit.status(TaskState::Failed).await?; @@ -163,3 +173,34 @@ pub(crate) fn dispatcher( ); Ok(JsonRpcDispatcher::new(handler)) } + +#[cfg(test)] +mod effects_tests { + #[tokio::test] + async fn a2a_cancelled_execution_retains_the_captured_owner() { + if crate::effects::isolated_test( + "protocols::a2a::effects_tests::a2a_cancelled_execution_retains_the_captured_owner", + ) { + return; + } + let observations = crate::effects::Observations::local_session(); + let executing = observations.clone(); + let result = async move { + executing.invocation_started(); + tokio::task::yield_now().await; + executing.invocation_completed(); + Err::<(), _>(agentkit_loop::LoopError::Cancelled) + } + .await; + super::record_failure("a2a-effects", &result.unwrap_err(), &observations); + let record = crate::effects::test_record("a2a-effects"); + assert_eq!(record["surface"], "a2a"); + assert_eq!(record["kind"], "cancelled"); + assert_eq!(record["possible_effects"]["source"], "local_session"); + assert_eq!( + record["possible_effects"]["tool_execution_completion_reported"], + true + ); + assert_eq!(record["possible_effects"]["observation_incomplete"], true); + } +} diff --git a/src/protocols/acp.rs b/src/protocols/acp.rs index 4893ba68..d295a5be 100644 --- a/src/protocols/acp.rs +++ b/src/protocols/acp.rs @@ -1325,20 +1325,10 @@ impl Server { cancellation: handle.cancellation_handle(), response_attempt_replacement: true, }; - let driver = match self + let driver = self .runtime .start_acp_driver_with_initial(context, &mut claim, forked) - .await - { - Ok(driver) => driver, - Err(error) => { - return Err(record_acp_runtime_failure( - &session_id, - "session_start", - error, - )); - } - }; + .await?; let current = driver .adapter .selection() @@ -1705,6 +1695,7 @@ async fn session_actor(actor: SessionActor) { &integration, &mut driver, &activity, + &background_jobs.observations, ).await, Err(error) => Err(error), }; @@ -1724,6 +1715,7 @@ async fn session_actor(actor: SessionActor) { &integration, &mut driver, &activity, + &background_jobs.observations, ).await; if let Err(error) = result { eprintln!("autonomous ACP continuation failed for {session_id}: {error}"); @@ -1931,12 +1923,27 @@ fn record_acp_runtime_failure( session_id: &agentkit_acp::SessionId, code: &str, error: impl ToString, +) -> AcpRuntimeError { + record_acp_runtime_failure_observed( + session_id, + code, + error, + &crate::effects::Observations::default(), + ) +} + +fn record_acp_runtime_failure_observed( + session_id: &agentkit_acp::SessionId, + code: &str, + error: impl ToString, + observations: &crate::effects::Observations, ) -> AcpRuntimeError { let rendered = error.to_string(); - match crate::fatal::record_runtime_error( + match crate::fatal::record_runtime_error_with_effects( &session_id.to_string(), crate::fatal::Surface::Acp, code, + observations.snapshot(), ) { Ok(path) => AcpRuntimeError::Loop(format!("{rendered}; fatal log: {}", path.display())), Err(log_error) => { @@ -1949,12 +1956,14 @@ fn record_acp_runtime_failure( fn record_acp_loop_failure( session_id: &agentkit_acp::SessionId, error: &LoopError, + observations: &crate::effects::Observations, ) -> AcpRuntimeError { let rendered = crate::fatal::render_loop_error(error); - match crate::fatal::record_loop_error( + match crate::fatal::record_loop_error_with_effects( &session_id.to_string(), crate::fatal::Surface::Acp, error, + observations.snapshot(), ) { Ok(Some(path)) => { AcpRuntimeError::Loop(format!("{rendered}; fatal log: {}", path.display())) @@ -1988,11 +1997,14 @@ async fn drive_prompt( skill_catalog .submit(skills, items, |items| driver.submit_input(items)) .map_err(|error| match error { - skill_catalog::SubmitError::Catalog(error) => { - record_acp_runtime_failure(session_id, "skill_catalog", error) - } + skill_catalog::SubmitError::Catalog(error) => record_acp_runtime_failure_observed( + session_id, + "skill_catalog", + error, + &background_jobs.observations, + ), skill_catalog::SubmitError::Submit(error) => { - record_acp_loop_failure(session_id, &error) + record_acp_loop_failure(session_id, &error, &background_jobs.observations) } })?; drive_submitted_prompt( @@ -2019,22 +2031,38 @@ async fn drive_runtime_prompt( structured_completion: bool, ) -> Result { if structured_completion { - let _ = settle_background_jobs(tasks, background_jobs).await?; + let _ = settle_background_jobs(tasks, background_jobs) + .await + .map_err(|error| { + record_acp_runtime_failure_observed( + session_id, + "prompt_preparation_settlement", + error, + &background_jobs.observations, + ) + })?; } - let current = runtime - .current_skills() - .await - .map_err(AcpRuntimeError::Loop)?; + let current = runtime.current_skills().await.map_err(|error| { + record_acp_runtime_failure_observed( + session_id, + "skill_refresh", + error, + &background_jobs.observations, + ) + })?; background_jobs.begin_turn(); let items = integration.input_port().prompt_to_items(&request)?; skill_catalog .submit(¤t.skills, items, |items| driver.submit_input(items)) .map_err(|error| match error { - skill_catalog::SubmitError::Catalog(error) => { - record_acp_runtime_failure(session_id, "skill_catalog", error) - } + skill_catalog::SubmitError::Catalog(error) => record_acp_runtime_failure_observed( + session_id, + "skill_catalog", + error, + &background_jobs.observations, + ), skill_catalog::SubmitError::Submit(error) => { - record_acp_loop_failure(session_id, &error) + record_acp_loop_failure(session_id, &error, &background_jobs.observations) } })?; drop(current); @@ -2044,6 +2072,7 @@ async fn drive_runtime_prompt( driver, true, structured_completion.then_some((tasks, background_jobs)), + &background_jobs.observations, ) .await } @@ -2073,11 +2102,12 @@ async fn drive_unsolicited( integration: &AcpIntegration, driver: &mut LoopDriver, activity: &activity::SessionActivity, + observations: &crate::effects::Observations, ) -> Result<(), AcpRuntimeError> { activity .execute( activity::ExecutionOrigin::Autonomous, - drive_finalized(session_id, integration, driver, false, None), + drive_finalized(session_id, integration, driver, false, None, observations), |reason| Some(reason.clone()), ) .await @@ -2102,8 +2132,19 @@ async fn drive_until_pause( answer_prompt: bool, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, ) -> Result, AcpRuntimeError> { - let reason = - drive_finalized(session_id, integration, driver, answer_prompt, structured).await?; + let fallback = crate::effects::Observations::local_session(); + let observations = structured + .map(|(_, jobs)| &jobs.observations) + .unwrap_or(&fallback); + let reason = drive_finalized( + session_id, + integration, + driver, + answer_prompt, + structured, + observations, + ) + .await?; if answer_prompt { Ok(Some(PromptResponse::new( agentkit_acp::finish_reason_to_stop_reason(&reason)?, @@ -2119,18 +2160,51 @@ async fn drive_finalized( driver: &mut LoopDriver, answer_prompt: bool, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, + observations: &crate::effects::Observations, ) -> Result { let cancellation = integration.cancellation_handle(session_id)?; let generation = cancellation.generation(); - let result = - drive_domain_until_pause(session_id, integration, driver, answer_prompt, structured).await; - activity::finalize( + let mut failure_recorded = false; + let result = drive_domain_until_pause( + session_id, + integration, + driver, + answer_prompt, + structured, + observations, + &mut failure_recorded, + ) + .await; + let result = activity::finalize( activity::ExecutionOutcome::new(result, cancellation.is_cancelled_since(generation)), structured, integration.flush_session_updates(session_id), - |_| Ok(()), + |_, origin| { + if (!failure_recorded || origin == activity::FailureOrigin::Finalization) + && let Err(error) = crate::fatal::record_runtime_error_with_effects( + &session_id.to_string(), + crate::fatal::Surface::Acp, + "session_finalization", + observations.snapshot(), + ) + { + tracing::warn!(%error, "could not record finalization observations"); + } + Ok(()) + }, ) - .await + .await; + if matches!(result, Ok(FinishReason::Cancelled)) + && let Err(error) = crate::fatal::record_loop_error_with_effects( + &session_id.to_string(), + crate::fatal::Surface::Acp, + &LoopError::Cancelled, + observations.snapshot(), + ) + { + tracing::warn!(%error, "could not record cancellation observations"); + } + result } async fn drive_domain_until_pause( @@ -2139,6 +2213,8 @@ async fn drive_domain_until_pause( driver: &mut LoopDriver, answer_prompt: bool, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, + observations: &crate::effects::Observations, + failure_recorded: &mut bool, ) -> Result { let cancellation = integration.cancellation_handle(session_id)?; let generation = cancellation.generation(); @@ -2148,13 +2224,16 @@ async fn drive_domain_until_pause( Err(LoopError::Cancelled) => { return Ok(FinishReason::Cancelled); } - Err(error) => return Err(record_acp_loop_failure(session_id, &error)), + Err(error) => { + *failure_recorded = true; + return Err(record_acp_loop_failure(session_id, &error, observations)); + } }; if cancellation.is_cancelled_since(generation) { - driver - .retire_interrupted_turn() - .await - .map_err(|error| record_acp_loop_failure(session_id, &error))?; + driver.retire_interrupted_turn().await.map_err(|error| { + *failure_recorded = true; + record_acp_loop_failure(session_id, &error, observations) + })?; return Ok(FinishReason::Cancelled); } match step { @@ -2731,7 +2810,7 @@ pub(super) mod tests { ), None, content.flush(), - |_| Ok(()), + |_, _| Ok(()), ) .await }, @@ -4009,6 +4088,11 @@ pub(super) mod tests { #[tokio::test] async fn live_prompt_boundary_refreshes_plugin_skill_catalog() { + if crate::effects::isolated_test( + "protocols::acp::tests::live_prompt_boundary_refreshes_plugin_skill_catalog", + ) { + return; + } let root = tempfile::tempdir().unwrap(); let config = root.path().join("config.toml"); std::fs::write(&config, "").unwrap(); @@ -4100,6 +4184,7 @@ pub(super) mod tests { .unwrap(); let task_manager = AsyncTaskManager::new(); let tasks = task_manager.handle(); + let background_jobs = BackgroundJobs::default(); let response = drive_runtime_prompt( &acp_session_id, &runtime, @@ -4113,13 +4198,52 @@ pub(super) mod tests { )], ), &tasks, - &BackgroundJobs::default(), + &background_jobs, false, ) .await .unwrap(); assert_eq!(response, FinishReason::Completed); assert_eq!(notification_items_seen.load(Ordering::SeqCst), 1); + // Retained facts from earlier work must survive next-prompt preparation failure. + background_jobs.observations.invocation_started(); + std::fs::write(&config, "invalid = [").unwrap(); + let error = drive_runtime_prompt( + &acp_session_id, + &runtime, + &integration, + &mut skill_catalog, + &mut driver, + PromptRequest::new( + acp_session_id.clone(), + vec![agentkit_acp::ContentBlock::Text( + agentkit_acp::TextContent::new("next prompt"), + )], + ), + &tasks, + &background_jobs, + false, + ) + .await + .unwrap_err(); + assert_eq!(error.to_string().matches("fatal log:").count(), 1); + let record = crate::effects::test_record(&acp_session_id.to_string()); + assert_eq!(record["code"], "skill_refresh"); + assert_eq!(record["possible_effects"]["source"], "local_session"); + assert_eq!( + record["possible_effects"]["tool_execution_start_reported"], + true + ); + assert_eq!( + std::fs::read_dir( + std::path::PathBuf::from(std::env::var_os("HOME").unwrap()) + .join(".kit/errors") + .join(acp_session_id.to_string()) + ) + .unwrap() + .count(), + 1 + ); drain.abort(); } @@ -4298,9 +4422,15 @@ pub(super) mod tests { .unwrap(); for _ in 0..2 { - drive_unsolicited(&session_id, &integration, &mut driver, &activity) - .await - .unwrap(); + drive_unsolicited( + &session_id, + &integration, + &mut driver, + &activity, + &crate::effects::Observations::local_session(), + ) + .await + .unwrap(); } assert_eq!(turns.load(Ordering::SeqCst), 2); assert!(states.try_recv().is_err()); @@ -4308,18 +4438,30 @@ pub(super) mod tests { driver .submit_input(vec![Item::notification("background result")]) .unwrap(); - drive_unsolicited(&session_id, &integration, &mut driver, &activity) - .await - .unwrap(); + drive_unsolicited( + &session_id, + &integration, + &mut driver, + &activity, + &crate::effects::Observations::local_session(), + ) + .await + .unwrap(); let started = states.try_recv().unwrap(); let ended = states.try_recv().unwrap(); assert!(started.active && !ended.active); assert_eq!(started.turn_id, 1); assert_eq!(started.turn_id, ended.turn_id); assert!(ended.error.is_none()); - drive_unsolicited(&session_id, &integration, &mut driver, &activity) - .await - .unwrap(); + drive_unsolicited( + &session_id, + &integration, + &mut driver, + &activity, + &crate::effects::Observations::local_session(), + ) + .await + .unwrap(); assert_eq!(turns.load(Ordering::SeqCst), 3); assert!(states.try_recv().is_err()); @@ -4753,6 +4895,108 @@ pub(super) mod tests { } } + #[tokio::test] + async fn startup_failure_records_once_with_original_classification_and_source() { + if crate::effects::isolated_test( + "protocols::acp::tests::startup_failure_records_once_with_original_classification_and_source", + ) { + return; + } + for before_owner in [false, true] { + let root = tempfile::tempdir().unwrap(); + let other_root = tempfile::tempdir().unwrap(); + let session_id = crate::session::new_id(); + let runtime = Runtime::with_session_provider_credentials_effort_and_openrouter_key( + root.path(), + "test/model", + crate::ProviderKind::OpenRouter, + crate::runtime::SessionRequest { + id: session_id.clone(), + resume: false, + force: false, + }, + crate::credentials::CredentialStorage::Memory, + None, + // SelectableAdapter accepts the selection, but starting its + // model session fails deterministically without network I/O. + Some(crate::provider::OpenRouterApiKey::new("")), + ) + .unwrap(); + let workspace = if before_owner { + other_root.path() + } else { + root.path() + } + .to_path_buf(); + let (client_transport, agent_transport) = Channel::duplex(); + let server = tokio::spawn(serve_transport(runtime, agent_transport)); + agent_client_protocol::Client + .builder() + .connect_with(client_transport, async move |connection| { + connection + .send_request(InitializeRequest::new(ProtocolVersion::V1)) + .block_task() + .await?; + let error = connection + .send_request(NewSessionRequest::new(workspace)) + .block_task() + .await + .expect_err("startup must fail"); + let rendered = error.to_string(); + assert_eq!(rendered.matches("fatal log:").count(), 1, "{rendered}"); + assert!( + rendered.contains(if before_owner { + "this Kit runtime is fixed to" + } else { + "--openrouter-api-key cannot be empty" + }), + "{rendered}" + ); + Ok(()) + }) + .await + .unwrap(); + server.abort(); + let _ = server.await; + + let directory = PathBuf::from(std::env::var_os("HOME").unwrap()) + .join(".kit/errors") + .join(&session_id); + let records: Vec<_> = std::fs::read_dir(directory) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "json")) + .collect(); + assert_eq!(records.len(), 1, "startup must record exactly once"); + let record: serde_json::Value = + serde_json::from_slice(&std::fs::read(&records[0]).unwrap()).unwrap(); + assert_eq!(record["surface"], "acp"); + assert_eq!(record["kind"], "runtime"); + assert_eq!( + record["code"], + if before_owner { + "session_start" + } else { + "invalid_state" + } + ); + assert_eq!( + record["possible_effects"]["source"], + if before_owner { + "unknown" + } else { + "local_session" + } + ); + assert_eq!(record["possible_effects"]["observation_incomplete"], true); + assert_eq!( + record["possible_effects"]["tool_execution_start_reported"], + false + ); + assert!(crate::session::load(root.path(), &session_id).is_err()); + } + } + #[tokio::test] async fn registry_shutdown_closes_real_session_and_rejects_reattach() { let root = tempfile::tempdir().unwrap(); @@ -5057,4 +5301,89 @@ pub(super) mod tests { server.abort(); let _ = server.await; } + #[tokio::test] + async fn root_cancellation_keeps_owner_in_unstructured_and_unsolicited_drives() { + if crate::effects::isolated_test( + "protocols::acp::tests::root_cancellation_keeps_owner_in_unstructured_and_unsolicited_drives", + ) { + return; + } + for autonomous in [false, true] { + let session_id = agentkit_acp::SessionId::new(if autonomous { + "effects-autonomous" + } else { + "effects-prompt" + }); + let loop_id = AgentkitSessionId::new(session_id.to_string()); + let integration = AcpIntegration::builder() + .name("effects-test") + .approval_resolver(AutoDenyResolver) + .build() + .unwrap(); + let (client, mut messages) = AcpClientHandle::channel(); + integration + .bind_session(AcpSessionBinding::new( + session_id.clone(), + loop_id.clone(), + client, + )) + .unwrap(); + let drain = tokio::spawn(async move { + while let Some(message) = messages.recv().await { + if let AcpClientMessage::Flush { response } = message { + let _ = response.send(()); + } + } + }); + let observations = crate::effects::Observations::local_session(); + // Evidence from a local background invocation in this live session. + observations.invocation_started(); + let mut driver = Agent::builder() + .model(CancelAdapter) + .observer(observations.clone()) + .input(vec![Item::text(ItemKind::User, "cancel")]) + .build() + .unwrap() + .start(SessionConfig::new(loop_id).without_cache()) + .await + .unwrap(); + if autonomous { + let (notifications, _) = mpsc::unbounded_channel(); + let activity = test_activity(session_id.clone(), notifications); + drive_unsolicited( + &session_id, + &integration, + &mut driver, + &activity, + &observations, + ) + .await + .unwrap(); + } else { + let reason = drive_finalized( + &session_id, + &integration, + &mut driver, + true, + None, + &observations, + ) + .await + .unwrap(); + assert_eq!(reason, FinishReason::Cancelled); + } + let record = crate::effects::test_record(&session_id.to_string()); + assert_eq!(record["kind"], "cancelled"); + assert_eq!(record["possible_effects"]["source"], "local_session"); + assert_eq!( + record["possible_effects"]["tool_execution_start_reported"], + true + ); + assert_eq!( + record["possible_effects"]["tool_execution_completion_reported"], + false + ); + drain.abort(); + } + } } diff --git a/src/protocols/acp/activity.rs b/src/protocols/acp/activity.rs index e7672a3e..1dce072c 100644 --- a/src/protocols/acp/activity.rs +++ b/src/protocols/acp/activity.rs @@ -164,6 +164,13 @@ impl ExecutionOutcome { } } +/// Identifies whether finalization replaced the execution's original error. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum FailureOrigin { + Execution, + Finalization, +} + /// Shared finalization order: cancel/drain structured work, drain all content even /// on failure, render a diagnostic, then return the outcome for single settlement. /// Hooks are transport only; neither hook chooses lifecycle or cleanup policy. @@ -174,22 +181,25 @@ pub(super) async fn finalize( &crate::runtime::BackgroundJobs, )>, flush: impl std::future::Future>, - diagnostic: impl FnOnce(&AcpRuntimeError) -> Result<(), AcpRuntimeError>, + diagnostic: impl FnOnce(&AcpRuntimeError, FailureOrigin) -> Result<(), AcpRuntimeError>, ) -> Result { let mut result = outcome.result; + let mut origin = FailureOrigin::Execution; if (result.is_err() || matches!(result, Ok(FinishReason::Cancelled))) && let Some((tasks, jobs)) = structured { super::cancel_background_jobs(tasks, jobs).await; if let Err(error) = super::settle_background_jobs(tasks, jobs).await { result = Err(error); + origin = FailureOrigin::Finalization; } } if let Err(error) = flush.await { result = Err(error); + origin = FailureOrigin::Finalization; } if let Err(error) = &result { - diagnostic(error)?; + diagnostic(error, origin)?; } result } @@ -296,7 +306,7 @@ mod tests { ExecutionOutcome::new(Err(AcpRuntimeError::Loop("provider failed".into())), true), None, async { Ok(()) }, - |_| panic!("cancelled model failure is not an error"), + |_, _| panic!("cancelled model failure is not an error"), ) .await .unwrap(); @@ -323,7 +333,8 @@ mod tests { order.lock().unwrap().push("flush"); Err(AcpRuntimeError::ClientClosed) }, - |error| { + |error, origin| { + assert_eq!(origin, FailureOrigin::Finalization); order.lock().unwrap().push("diagnostic"); diagnostic.lock().unwrap().push(error.to_string()); Ok(()) @@ -343,6 +354,37 @@ mod tests { assert_eq!(diagnostic.lock().unwrap().len(), 1); } + #[tokio::test] + async fn finalization_marks_replacement_of_an_already_recorded_failure() { + for flush_fails in [false, true] { + let result = finalize( + ExecutionOutcome::new(Err(AcpRuntimeError::Loop("driver failed".into())), false), + None, + async { + if flush_fails { + Err(AcpRuntimeError::ClientClosed) + } else { + Ok(()) + } + }, + |error, origin| { + assert_eq!( + origin, + if flush_fails { + FailureOrigin::Finalization + } else { + FailureOrigin::Execution + } + ); + assert_eq!(matches!(error, AcpRuntimeError::ClientClosed), flush_fails); + Ok(()) + }, + ) + .await; + assert!(result.is_err()); + } + } + #[tokio::test] async fn failed_terminal_projection_is_not_retried() { let calls = Arc::new(Mutex::new(0)); diff --git a/src/protocols/acp/v2.rs b/src/protocols/acp/v2.rs index eb923cb5..db6e3c7e 100644 --- a/src/protocols/acp/v2.rs +++ b/src/protocols/acp/v2.rs @@ -119,23 +119,40 @@ fn list_sessions_error(error: ListSessionsError) -> agent_client_protocol::Error } } +#[cfg(test)] fn map_loop_error(session_id: &wire::SessionId, error: &LoopError) -> AcpRuntimeError { + map_loop_error_observed(session_id, error, &crate::effects::Observations::default()) +} +#[cfg(test)] +fn loop_error_stop_reason( + session_id: &wire::SessionId, + error: &LoopError, +) -> Result { + loop_error_stop_reason_observed(session_id, error, &crate::effects::Observations::default()) +} + +fn map_loop_error_observed( + session_id: &wire::SessionId, + error: &LoopError, + observations: &crate::effects::Observations, +) -> AcpRuntimeError { if matches!(error, LoopError::Cancelled) { AcpRuntimeError::Cancelled } else { let session_id = agentkit_acp::SessionId::new(session_id.to_string()); - super::record_acp_loop_failure(&session_id, error) + super::record_acp_loop_failure(&session_id, error, observations) } } -fn loop_error_stop_reason( +fn loop_error_stop_reason_observed( session_id: &wire::SessionId, error: &LoopError, + observations: &crate::effects::Observations, ) -> Result { if matches!(error, LoopError::Cancelled) { Ok(FinishReason::Cancelled) } else { - Err(map_loop_error(session_id, error)) + Err(map_loop_error_observed(session_id, error, observations)) } } @@ -1193,8 +1210,8 @@ async fn session_actor(actor: SessionActor) &busy, &mut driver, &sink, - &activity,).await, - Err(error) => Err(map_loop_error(&session_id, &error)), + &activity, &background_jobs.observations,).await, + Err(error) => Err(map_loop_error_observed(&session_id, &error, &background_jobs.observations)), }; if let Err(error) = result { eprintln!("ACP v2 autonomous turn failed for {session_id}: {error}"); @@ -1212,7 +1229,7 @@ async fn session_actor(actor: SessionActor) &busy, &mut driver, &sink, - &activity,).await + &activity, &background_jobs.observations,).await { eprintln!("ACP v2 autonomous turn failed for {session_id}: {error}"); } @@ -1257,6 +1274,12 @@ async fn prepare_prompt( && let Err(error) = super::settle_background_jobs(tasks, background_jobs).await { handle.stop_injection_turn(); + let error = super::record_acp_runtime_failure_observed( + &agentkit_acp::SessionId::new(session_id.to_string()), + "prompt_preparation_settlement", + error, + &background_jobs.observations, + ); let _ = reply.send(Err(error)); return Ok(()); } @@ -1269,7 +1292,13 @@ async fn prepare_prompt( Ok(current) => current, Err(error) => { handle.stop_injection_turn(); - let _ = reply.send(Err(AcpRuntimeError::Loop(error))); + let error = super::record_acp_runtime_failure_observed( + &agentkit_acp::SessionId::new(session_id.to_string()), + "skill_refresh", + error, + &background_jobs.observations, + ); + let _ = reply.send(Err(error)); return Ok(()); } }; @@ -1282,9 +1311,16 @@ async fn prepare_prompt( .submit(skills, items, |items| driver.submit_input(items)) .map_err(|error| match error { skill_catalog::SubmitError::Catalog(error) => { - AcpRuntimeError::Loop(format!("skill catalog error: {error}")) + super::record_acp_runtime_failure_observed( + &agentkit_acp::SessionId::new(session_id.to_string()), + "skill_catalog", + error, + &background_jobs.observations, + ) + } + skill_catalog::SubmitError::Submit(error) => { + map_loop_error_observed(session_id, &error, &background_jobs.observations) } - skill_catalog::SubmitError::Submit(error) => map_loop_error(session_id, &error), })?; integration.begin_prompt(session_id) }); @@ -1321,6 +1357,7 @@ async fn prepare_prompt( structured_completion.then_some((tasks, background_jobs)), activity, ExecutionOrigin::Prompt, + &background_jobs.observations, ) .await } @@ -1360,6 +1397,7 @@ impl TurnControl for AcpSessionHandle { } } +#[cfg(test)] async fn drive_prompt( session_id: &wire::SessionId, driver: &mut LoopDriver, @@ -1367,6 +1405,33 @@ async fn drive_prompt( cancellation_generation: u64, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, ) -> Result +where + S: ModelSession + Send + 'static, + C: TurnControl, +{ + let observations = crate::effects::Observations::local_session(); + let mut recorded = false; + drive_prompt_observed( + session_id, + driver, + control, + cancellation_generation, + structured, + &observations, + &mut recorded, + ) + .await +} + +async fn drive_prompt_observed( + session_id: &wire::SessionId, + driver: &mut LoopDriver, + control: &C, + cancellation_generation: u64, + structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, + observations: &crate::effects::Observations, + failure_recorded: &mut bool, +) -> Result where S: ModelSession + Send + 'static, C: TurnControl, @@ -1377,6 +1442,8 @@ where control, cancellation_generation, structured, + observations, + failure_recorded, ) .await; if matches!(result, Ok(FinishReason::Cancelled)) { @@ -1396,6 +1463,8 @@ async fn drive_prompt_inner( control: &C, cancellation_generation: u64, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, + observations: &crate::effects::Observations, + failure_recorded: &mut bool, ) -> Result where S: ModelSession + Send + 'static, @@ -1409,7 +1478,8 @@ where if control.is_cancelled_since(cancellation_generation) { return Ok(FinishReason::Cancelled); } - return loop_error_stop_reason(session_id, &error); + *failure_recorded = !matches!(error, LoopError::Cancelled); + return loop_error_stop_reason_observed(session_id, &error, observations); } }; if control.is_cancelled_since(cancellation_generation) { @@ -1497,7 +1567,8 @@ where if control.is_cancelled_since(cancellation_generation) { return Ok(FinishReason::Cancelled); } - return loop_error_stop_reason(session_id, &error); + *failure_recorded = !matches!(error, LoopError::Cancelled); + return loop_error_stop_reason_observed(session_id, &error, observations); } } } @@ -1515,17 +1586,21 @@ async fn run_active_turn( structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, activity: &SessionActivity, origin: ExecutionOrigin, + observations: &crate::effects::Observations, ) -> Result<(), AcpRuntimeError> { activity .execute( origin, async { - let result = drive_prompt( + let mut failure_recorded = false; + let result = drive_prompt_observed( session_id, driver, handle, cancellation_generation, structured, + observations, + &mut failure_recorded, ) .await; let outcome = super::activity::ExecutionOutcome::new( @@ -1539,9 +1614,31 @@ async fn run_active_turn( outcome, structured, integration.flush_session_updates(session_id), - |error| sink.update(error_diagnostic_notification(session_id, error)), + |error, origin| { + if (!failure_recorded || origin == super::activity::FailureOrigin::Finalization) + && let Err(log_error) = crate::fatal::record_runtime_error_with_effects( + &session_id.to_string(), + crate::fatal::Surface::Acp, + "session_finalization", + observations.snapshot(), + ) + { + tracing::warn!(%log_error, "could not record v2 finalization observations"); + } + sink.update(error_diagnostic_notification(session_id, error)) + }, ) .await; + if matches!(result, Ok(FinishReason::Cancelled)) + && let Err(error) = crate::fatal::record_loop_error_with_effects( + &session_id.to_string(), + crate::fatal::Surface::Acp, + &LoopError::Cancelled, + observations.snapshot(), + ) + { + tracing::warn!(%error, "could not record v2 cancellation observations"); + } integration.finish_prompt(session_id); result }, @@ -1551,6 +1648,7 @@ async fn run_active_turn( .map(|_| ()) } +#[allow(clippy::too_many_arguments)] async fn drive_autonomous( session_id: &wire::SessionId, integration: &AcpIntegration, @@ -1559,6 +1657,7 @@ async fn drive_autonomous( driver: &mut LoopDriver, sink: &ResponseReplacementSink, activity: &SessionActivity, + observations: &crate::effects::Observations, ) -> Result<(), AcpRuntimeError> { if claim_prompt(busy).is_err() { return Ok(()); @@ -1577,6 +1676,7 @@ async fn drive_autonomous( None, activity, ExecutionOrigin::Autonomous, + observations, ) .await; integration.finish_prompt(session_id); @@ -3134,6 +3234,7 @@ mod tests { handle.start_injection_turn(); let generation = handle.cancellation_handle().generation(); let background_jobs = BackgroundJobs::default(); + let observations = crate::effects::Observations::local_session(); let prompt = run_active_turn( &session_id, &integration, @@ -3144,6 +3245,7 @@ mod tests { Some((&tasks, &background_jobs)), &activity, ExecutionOrigin::Prompt, + &observations, ); tokio::pin!(prompt); @@ -3461,6 +3563,7 @@ mod tests { &mut driver, &sink, &activity, + &crate::effects::Observations::local_session(), ) .await .unwrap(); @@ -3518,6 +3621,7 @@ mod tests { &mut driver, &sink, &activity, + &crate::effects::Observations::local_session(), ) .await .unwrap(); @@ -3535,6 +3639,7 @@ mod tests { &mut driver, &sink, &activity, + &crate::effects::Observations::local_session(), ) .await .unwrap(); @@ -3601,6 +3706,7 @@ mod tests { &mut driver, &sink, &activity, + &crate::effects::Observations::local_session(), ) .await; @@ -3675,6 +3781,7 @@ mod tests { &mut driver, &sink, &activity, + &crate::effects::Observations::local_session(), ) .await .unwrap(); @@ -3732,6 +3839,7 @@ mod tests { &mut driver, &sink, &activity, + &crate::effects::Observations::local_session(), ) .await; assert!(matches!(result, Err(AcpRuntimeError::ClientClosed))); @@ -3801,6 +3909,7 @@ mod tests { &mut driver, &sink, &activity, + &crate::effects::Observations::local_session(), ) .await; @@ -4377,4 +4486,199 @@ mod tests { Err(ListSessionsError::InvalidCursor) )); } + #[tokio::test] + async fn root_stream_cancellation_records_the_same_local_owner() { + if crate::effects::isolated_test( + "protocols::acp::v2::tests::root_stream_cancellation_records_the_same_local_owner", + ) { + return; + } + let integration = AcpIntegration::default(); + let recording = RecordingSink::default(); + let sink = ResponseReplacementSink::new(recording.clone()); + let session_id = wire::SessionId::new("effects-v2-cancel"); + let loop_id = SessionId::new("effects-v2-loop"); + let activity = native_activity(session_id.clone(), sink.clone()); + let handle = integration + .bind_session( + AcpSessionBinding::new(session_id.clone(), loop_id.clone(), sink.clone()) + .cancellation(CancellationController::new()), + ) + .unwrap(); + let observations = crate::effects::Observations::local_session(); + let observer = ResponseReplacementObserver::new( + integration.clone(), + sink.clone(), + session_id.clone(), + activity.clone(), + ); + let mut driver = Agent::builder() + .model(StreamingCancellationAdapter { + interrupt: handle.clone(), + }) + .observer(observer) + .observer(observations.clone()) + .cancellation(handle.cancellation_handle()) + .build() + .unwrap() + .start(SessionConfig::new(loop_id).without_cache()) + .await + .unwrap(); + driver + .submit_input(vec![Item::text(ItemKind::User, "private prompt")]) + .unwrap(); + handle.prepare_injection_turn(); + let generation = handle.cancellation_handle().generation(); + handle.start_injection_turn(); + run_active_turn( + &session_id, + &integration, + &handle, + &mut driver, + &sink, + generation, + None, + &activity, + ExecutionOrigin::Prompt, + &observations, + ) + .await + .unwrap(); + let record = crate::effects::test_record(&session_id.to_string()); + assert_eq!(record["kind"], "cancelled"); + assert_eq!(record["possible_effects"]["source"], "local_session"); + assert_eq!( + record["possible_effects"]["assistant_output_observed"], + true + ); + assert_eq!( + record["possible_effects"]["tool_execution_start_reported"], + false + ); + assert!( + !serde_json::to_string(&record) + .unwrap() + .contains("private prompt") + ); + assert_running_then_idle( + &recording.updates.lock().unwrap(), + wire::StopReason::Cancelled, + ); + } + #[tokio::test] + async fn preparation_failure_retains_previous_prompt_observations() { + if crate::effects::isolated_test( + "protocols::acp::v2::tests::preparation_failure_retains_previous_prompt_observations", + ) { + return; + } + let root = tempfile::tempdir().unwrap(); + let config = root.path().join("config.toml"); + std::fs::write(&config, "").unwrap(); + let plugins = crate::plugins::PluginRuntime::load( + config.clone(), + root.path().to_path_buf(), + root.path().join("cache"), + root.path().join("data"), + ) + .await + .unwrap(); + let runtime = Runtime::with_plugin_runtime( + Runtime::new(root.path(), "gpt-5.4").unwrap(), + Some(plugins), + ) + .unwrap(); + let runtime = Runtime::with_mcp_config( + runtime, + None, + Vec::new(), + false, + crate::tools::mcp::CredentialStorage::Memory, + ) + .await + .unwrap(); + let baseline = runtime.current_skills().await.unwrap(); + let mut skill_catalog = skill_catalog::SkillCatalogMonitor::new(&baseline.skills).unwrap(); + drop(baseline); + let integration = AcpIntegration::default(); + let recording = RecordingSink::default(); + let sink = ResponseReplacementSink::new(recording.clone()); + let session_id = wire::SessionId::new("effects-v2-preparation"); + let loop_id = SessionId::new("effects-v2-preparation-loop"); + let activity = native_activity(session_id.clone(), sink.clone()); + let handle = integration + .bind_session(AcpSessionBinding::new( + session_id.clone(), + loop_id.clone(), + sink.clone(), + )) + .unwrap(); + let turns = Arc::new(AtomicU64::new(0)); + let mut driver = Agent::builder() + .model(TestAdapter { + outcome: TestOutcome::ProviderError, + turns: turns.clone(), + interrupt: None, + }) + .build() + .unwrap() + .start(SessionConfig::new(loop_id).without_cache()) + .await + .unwrap(); + let manager = AsyncTaskManager::new(); + let tasks = manager.handle(); + let jobs = BackgroundJobs::default(); + // A prior invocation's observations are cumulative, not attributed to this rejected prompt. + jobs.observations.invocation_started(); + jobs.begin_turn(); + std::fs::write(&config, "invalid = [").unwrap(); + handle.prepare_injection_turn(); + let (reply, response) = oneshot::channel(); + prepare_prompt( + &session_id, + PromptSkillSource::Runtime(&runtime), + &integration, + &handle, + &mut skill_catalog, + &mut driver, + PromptCommand { + request: wire::PromptRequest::new( + session_id.clone(), + vec![wire::ContentBlock::Text(wire::TextContent::new( + "next prompt", + ))], + ), + cancellation_generation: handle.cancellation_handle().generation(), + reply, + }, + &sink, + &tasks, + &jobs, + false, + &activity, + ) + .await + .unwrap(); + let error = response.await.unwrap().unwrap_err(); + assert_eq!(error.to_string().matches("fatal log:").count(), 1); + assert_eq!(turns.load(Ordering::Relaxed), 0); + assert!(recording.updates.lock().unwrap().is_empty()); + let record = crate::effects::test_record(&session_id.to_string()); + assert_eq!(record["code"], "skill_refresh"); + assert_eq!(record["possible_effects"]["source"], "local_session"); + assert_eq!( + record["possible_effects"]["tool_execution_start_reported"], + true + ); + assert_eq!( + std::fs::read_dir( + std::path::PathBuf::from(std::env::var_os("HOME").unwrap()) + .join(".kit/errors") + .join(session_id.to_string()) + ) + .unwrap() + .count(), + 1 + ); + } } diff --git a/src/runtime.rs b/src/runtime.rs index 6134f640..9a0be759 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -266,6 +266,19 @@ impl Drop for SessionClaim { } } +// Preserve startup classification until the single recording boundary, including +// failures that occur before a local observation owner has been created. +enum AcpStartupFailure { + Runtime(AcpRuntimeError), + Loop(LoopError, crate::effects::Observations), +} + +impl From for AcpStartupFailure { + fn from(error: AcpRuntimeError) -> Self { + Self::Runtime(error) + } +} + pub(crate) struct AcpDriver { pub driver: LoopDriver, pub skills: Vec, @@ -951,40 +964,83 @@ impl Runtime { skills: Arc, ) -> ComposeOnly { let mut children = agentkit_tools_core::ToolRegistry::new() - .with(Observed::new(ArtifactTool::new(crate::artifacts::base( - &self.root, - )))) - .with(Observed::new(DocsTool::new())) - .with(Observed::new(ShellTool::new(self.root.clone()))) - .with(Observed::new(EditTool::new(self.root.clone()))); + .with( + Observed::new(ArtifactTool::new(crate::artifacts::base(&self.root))) + .with_observations(background_jobs.observations.clone()), + ) + .with( + Observed::new(DocsTool::new()) + .with_observations(background_jobs.observations.clone()), + ) + .with( + Observed::new(ShellTool::new(self.root.clone())) + .with_observations(background_jobs.observations.clone()), + ) + .with( + Observed::new(EditTool::new(self.root.clone())) + .with_observations(background_jobs.observations.clone()), + ); if depth < self.max_subagent_depth { children - .register(Observed::new(SubagentTool::new(subagents.clone(), depth))) - .register(Observed::new(ForkTool::new(subagents.clone(), depth))); + .register( + Observed::new(SubagentTool::new(subagents.clone(), depth)) + .with_observations(background_jobs.observations.clone()), + ) + .register( + Observed::new(ForkTool::new(subagents.clone(), depth)) + .with_observations(background_jobs.observations.clone()), + ); } children - .register(Observed::new(PromptTool::new(subagents.clone()))) - .register(Observed::new(SubagentsTool::new(subagents.clone()))) - .register(Observed::new(CloseTool::new(subagents, { - let background_jobs = background_jobs.clone(); - move |call_id, allow_pending| { - if allow_pending { - background_jobs.cancel(call_id) - } else { - background_jobs.cancel_running(call_id) + .register( + Observed::new(PromptTool::new(subagents.clone())) + .with_observations(background_jobs.observations.clone()), + ) + .register( + Observed::new(SubagentsTool::new(subagents.clone())) + .with_observations(background_jobs.observations.clone()), + ) + .register( + Observed::new(CloseTool::new(subagents, { + let background_jobs = background_jobs.clone(); + move |call_id, allow_pending| { + if allow_pending { + background_jobs.cancel(call_id) + } else { + background_jobs.cancel_running(call_id) + } } - } - }))) - .register(Observed::new(A2aTool::new())) - .register(Observed::new(ToolSearch::new(self.mcp.clone()))) - .register(Observed::new(AuthTool::new(self.mcp.clone()))) - .register(Observed::new(McpTool::new(self.mcp.clone()))); + })) + .with_observations(background_jobs.observations.clone()), + ) + .register( + Observed::new(A2aTool::new()) + .with_observations(background_jobs.observations.clone()), + ) + .register( + Observed::new(ToolSearch::new(self.mcp.clone())) + .with_observations(background_jobs.observations.clone()), + ) + .register( + Observed::new(AuthTool::new(self.mcp.clone())) + .with_observations(background_jobs.observations.clone()), + ) + .register( + Observed::new(McpTool::new(self.mcp.clone())) + .with_observations(background_jobs.observations.clone()), + ); if let Some(skill_tool) = &self.dynamic_skill_tool { - children.register(observe_shared(Arc::clone(skill_tool))); + children.register(observe_shared( + Arc::clone(skill_tool), + background_jobs.observations.clone(), + )); } else { let skill_tools = skills.tool_registry(); if let Some(skill_tool) = skill_tools.get(&ToolName::new("skill")) { - children.register(observe_shared(skill_tool)); + children.register(observe_shared( + skill_tool, + background_jobs.observations.clone(), + )); } } let hidden_tools = children.clone(); @@ -1072,27 +1128,25 @@ impl Runtime { ) })?; let subagents = self.subagents.fresh(); + let background_jobs = BackgroundJobs::default(); let agent = Agent::builder() .model(self.adapter.clone()) .telemetry(self.agentkit_telemetry()) - .add_tool_source(self.compose_with_jobs( - 0, - subagents, - BackgroundJobs::default(), - skills, - )) + .add_tool_source(self.compose_with_jobs(0, subagents, background_jobs.clone(), skills)) .task_manager(background_task_manager()) .mutator(compactor) + .observer(background_jobs.observations.clone()) .transcript_observer(opened.observer) .transcript(opened.transcript) .input(vec![Item::text(ItemKind::User, prompt)]) .build() .map_err(|error| { - record_runtime_failure( + record_runtime_failure_observed( &session_id, crate::fatal::Surface::Prompt, "agent_build", error.to_string(), + &background_jobs.observations, ) })?; let mut driver = match agent @@ -1110,6 +1164,7 @@ impl Runtime { &session_id, crate::fatal::Surface::Prompt, &error, + &background_jobs.observations, )); } }; @@ -1119,6 +1174,7 @@ impl Runtime { &session_id, crate::fatal::Surface::Prompt, &error, + &background_jobs.observations, )), } } @@ -1128,6 +1184,22 @@ impl Runtime { prompt: String, depth: usize, cancellation: Option, + ) -> Result { + self.run_cancelled_observed( + prompt, + depth, + cancellation, + crate::effects::Observations::local_session(), + ) + .await + } + + pub(crate) async fn run_cancelled_observed( + self: &Arc, + prompt: String, + depth: usize, + cancellation: Option, + observations: crate::effects::Observations, ) -> Result { let controller = CancellationController::new(); let bridge = cancellation.map(|token| { @@ -1138,7 +1210,7 @@ impl Runtime { }) }); let result = self - .run_interruptible(prompt, depth, Some(controller.handle())) + .run_interruptible_observed(prompt, depth, Some(controller.handle()), observations) .await; if let Some(bridge) = bridge { bridge.abort(); @@ -1153,6 +1225,22 @@ impl Runtime { prompt: String, depth: usize, cancellation: Option, + ) -> Result { + self.run_interruptible_observed( + prompt, + depth, + cancellation, + crate::effects::Observations::local_session(), + ) + .await + } + + async fn run_interruptible_observed( + self: &Arc, + prompt: String, + depth: usize, + cancellation: Option, + observations: crate::effects::Observations, ) -> Result { if crate::resilient_fs::shutdown_token().is_cancelled() { return Err(LoopError::InvalidState( @@ -1178,17 +1266,17 @@ impl Runtime { ) .map_err(LoopError::InvalidState)?; let subagents = self.subagents.fresh(); + let background_jobs = BackgroundJobs { + observations: observations.clone(), + ..Default::default() + }; let builder = Agent::builder() .model(self.adapter.clone()) .telemetry(self.agentkit_telemetry()) - .add_tool_source(self.compose_with_jobs( - depth, - subagents, - BackgroundJobs::default(), - skills, - )) + .add_tool_source(self.compose_with_jobs(depth, subagents, background_jobs, skills)) .task_manager(background_task_manager()) .mutator(compactor) + .observer(observations) .transcript(transcript) .input(vec![Item::text(ItemKind::User, prompt)]); let builder = builder.cancellation(controller.handle()); @@ -1261,12 +1349,43 @@ impl Runtime { .await } + // All callers receive an already-recorded error. Do not record it again at + // a protocol boundary: that would discard the typed loop classification. pub(crate) async fn start_acp_driver_with_initial( self: &Arc, context: AcpDriverContext, claim: &mut SessionClaim, forked: Option, ) -> Result + where + I: LoopObserver + Clone + 'static, + { + self.prepare_acp_driver_with_initial(context, claim, forked) + .await + .map_err(|failure| { + AcpRuntimeError::Loop(match failure { + AcpStartupFailure::Runtime(error) => record_runtime_failure( + claim.id(), + crate::fatal::Surface::Acp, + "session_start", + error.to_string(), + ), + AcpStartupFailure::Loop(error, observations) => record_loop_failure( + claim.id(), + crate::fatal::Surface::Acp, + &error, + &observations, + ), + }) + }) + } + + async fn prepare_acp_driver_with_initial( + self: &Arc, + context: AcpDriverContext, + claim: &mut SessionClaim, + forked: Option, + ) -> Result where I: LoopObserver + Clone + 'static, { @@ -1276,7 +1395,8 @@ impl Runtime { return Err(AcpRuntimeError::Loop(format!( "this Kit runtime is fixed to {} and does not accept additional directories", self.root.display() - ))); + )) + .into()); } let request = claim.request.clone(); let session_id = request.id.clone(); @@ -1293,7 +1413,8 @@ impl Runtime { if request.resume { return Err(AcpRuntimeError::Loop( "a forked transcript requires a new session identity".into(), - )); + ) + .into()); } transcript } else if request.resume { @@ -1367,14 +1488,17 @@ impl Runtime { .task_manager(task_manager) .mutator(compactor) .observer(context.integration.as_ref().clone()) + .observer(background_jobs.observations.clone()) .transcript_observer(opened.observer) .transcript(opened.transcript) .cancellation(context.cancellation) .build() - .map_err(|error| AcpRuntimeError::Loop(error.to_string()))? + .map_err(|error| AcpStartupFailure::Loop(error, background_jobs.observations.clone()))? .start(session_config) .await - .map_err(|error| AcpRuntimeError::Loop(error.to_string()))?; + .map_err(|error| { + AcpStartupFailure::Loop(error, background_jobs.observations.clone()) + })?; let driver = AcpDriver { driver, skills: skill_catalog, @@ -1704,6 +1828,7 @@ pub(crate) struct BackgroundActivity { #[derive(Clone)] pub(crate) struct BackgroundJobs { + pub(crate) observations: crate::effects::Observations, state: Arc>, activity: watch::Sender, } @@ -1712,6 +1837,7 @@ impl Default for BackgroundJobs { fn default() -> Self { let (activity, _) = watch::channel(0); Self { + observations: crate::effects::Observations::local_session(), state: Arc::new(Mutex::new(BackgroundJobState::default())), activity, } @@ -2050,7 +2176,10 @@ impl Tool for BackgroundableCompose { crate::artifacts::directory(&self.root, &request.session_id.0, &call_id.0); let request = Self::sanitized(request)?; let _job = self.begin_background(background, &call_id, ctx); - match self.inner.invoke(request, ctx).await { + self.background_jobs.observations.invocation_started(); + let outcome = self.inner.invoke(request, ctx).await; + self.background_jobs.observations.invocation_completed(); + match outcome { Ok(mut result) => { match crate::compose_output::guard(&artifact_directory, result.result.output).await { @@ -2079,7 +2208,12 @@ impl Tool for BackgroundableCompose { Err(error) => return ToolExecutionOutcome::Failed(error), }; let _job = self.begin_background(background, &call_id, ctx); - match self.inner.invoke_outcome(request, ctx).await { + self.background_jobs.observations.invocation_started(); + let outcome = self.inner.invoke_outcome(request, ctx).await; + if !matches!(outcome, ToolExecutionOutcome::Interrupted(_)) { + self.background_jobs.observations.invocation_completed(); + } + match outcome { ToolExecutionOutcome::Completed(mut result) => { match crate::compose_output::guard(&artifact_directory, result.result.output).await { @@ -2333,7 +2467,28 @@ fn record_runtime_failure( code: &str, rendered: String, ) -> String { - match crate::fatal::record_runtime_error(session_id, surface, code) { + record_runtime_failure_observed( + session_id, + surface, + code, + rendered, + &crate::effects::Observations::default(), + ) +} + +fn record_runtime_failure_observed( + session_id: &str, + surface: crate::fatal::Surface, + code: &str, + rendered: String, + observations: &crate::effects::Observations, +) -> String { + match crate::fatal::record_runtime_error_with_effects( + session_id, + surface, + code, + observations.snapshot(), + ) { Ok(path) => format!("{rendered}; fatal log: {}", path.display()), Err(log_error) => { eprintln!("could not store fatal error log for {session_id}: {log_error}"); @@ -2346,9 +2501,15 @@ fn record_loop_failure( session_id: &str, surface: crate::fatal::Surface, error: &LoopError, + observations: &crate::effects::Observations, ) -> String { let rendered = crate::fatal::render_loop_error(error); - match crate::fatal::record_loop_error(session_id, surface, error) { + match crate::fatal::record_loop_error_with_effects( + session_id, + surface, + error, + observations.snapshot(), + ) { Ok(Some(path)) => format!("{rendered}; fatal log: {}", path.display()), Ok(None) => rendered, Err(log_error) => { diff --git a/src/runtime/tests.rs b/src/runtime/tests.rs index 37dc1498..4f673402 100644 --- a/src/runtime/tests.rs +++ b/src/runtime/tests.rs @@ -1722,3 +1722,90 @@ fn system_prompt_guides_compose_and_subagent_hygiene() { let max_depth_prompt = runtime.system_prompt(runtime.max_subagent_depth()); assert!(max_depth_prompt.contains("This task was delegated to you by the primary agent.")); } + +#[tokio::test] +async fn root_compose_receipts_preserve_both_entry_points_and_background_lifetime() { + for native in [false, true] { + for failing in [false, true] { + let root = tempfile::tempdir().unwrap(); + let runtime = Runtime::new(root.path(), "gpt-5.4").unwrap(); + let compose = runtime.compose(0); + let jobs = compose.backgroundable.background_jobs.clone(); + let source: Arc = Arc::new(compose.compose.clone()); + let executor: Arc = Arc::new(BasicToolExecutor::new([source])); + let session_id = SessionId::new("root-effects"); + let turn_id = TurnId::new("first"); + let permissions = Arc::new(AllowAllPermissions); + let resources: Arc = Arc::new(()); + let owned = OwnedToolContext { + session_id: session_id.clone(), + turn_id: turn_id.clone(), + metadata: MetadataMap::new(), + permissions: permissions.clone(), + resources: resources.clone(), + cancellation: None, + execution_scope: Some(ToolExecutionScope { + executor, + session_id: session_id.clone(), + turn_id: turn_id.clone(), + permissions, + resources, + cancellation: None, + }), + approved_request: None, + }; + let request = ToolRequest::new( + ToolCallId::new("effects-compose"), + ToolName::new("compose"), + json!({ + "script": if failing { "return fail(\"FAILED\", \"private failure text\")" } else { "return shell({ command: \"sleep 0.05\" })" }, "background": true, + }), + session_id, + turn_id, + ); + let invocation = async { + if native { + match compose + .backgroundable + .invoke_outcome(request, &mut owned.borrowed()) + .await + { + ToolExecutionOutcome::Completed(_) => assert!(!failing), + ToolExecutionOutcome::Failed(_) => assert!(failing), + other => panic!("unexpected native outcome: {other:?}"), + } + } else { + assert_eq!( + compose + .backgroundable + .invoke(request, &mut owned.borrowed()) + .await + .is_err(), + failing + ); + } + }; + let later_prompt = async { + tokio::time::timeout(Duration::from_secs(2), async { + while !jobs.observations.snapshot().tool_execution_start_reported { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + jobs.begin_turn(); + assert!(jobs.observations.snapshot().tool_execution_start_reported); + }; + tokio::join!(invocation, later_prompt); + let effects = jobs.observations.snapshot(); + assert_eq!( + effects.source, + crate::effects::ObservationSource::LocalSession + ); + assert!(effects.tool_execution_start_reported); + assert!(effects.tool_execution_completion_reported); + assert!(effects.observation_incomplete); + assert!(!serde_json::to_string(&effects).unwrap().contains("private")); + } + } +} diff --git a/src/tools/observed.rs b/src/tools/observed.rs index bf604073..6703577d 100644 --- a/src/tools/observed.rs +++ b/src/tools/observed.rs @@ -1,34 +1,60 @@ -//! Lifecycle reporting for the hidden tools behind `compose`. -//! -//! The wrapper is transparent to the model and to compose: it forwards the -//! spec, permission requests, and invocation untouched, and only publishes -//! start/finish events on the runtime side channel (see [`crate::events`]) so -//! a client can draw what a Runlet program is doing while it runs. +//! Transparent lifecycle reporting for the hidden tools behind `compose`. +//! Effects observations are independent of the opt-in stderr display channel. use std::{sync::Arc, time::Instant}; use agentkit_core::ToolOutput; use agentkit_tools_core::{ - PermissionRequest, Tool, ToolContext, ToolError, ToolRequest, ToolResult, ToolSpec, + PermissionRequest, Tool, ToolContext, ToolError, ToolExecutionOutcome, ToolRequest, ToolResult, + ToolSpec, }; use async_trait::async_trait; - use serde_json::{Value, json}; -use crate::events::{self, RuntimeEvent, summarize_input, summarize_output}; +use crate::{ + effects::Observations, + events::{self, RuntimeEvent, summarize_input, summarize_output}, +}; /// Wraps a tool so its calls appear on the runtime side channel. -pub struct Observed(T); +pub struct Observed { + tool: T, + observations: Option, +} impl Observed { pub const fn new(tool: T) -> Self { - Self(tool) + Self { + tool, + observations: None, + } + } + + pub(crate) fn with_observations(mut self, observations: Observations) -> Self { + self.observations = Some(observations); + self + } + + fn start(&self, request: &ToolRequest) -> Option { + if let Some(observations) = &self.observations { + observations.invocation_started(); + } + DisplayInvocation::start(request) + } + + fn finish(&self, display: Option, result: Result<&ToolResult, &ToolError>) { + if let Some(observations) = &self.observations { + observations.invocation_completed(); + } + if let Some(display) = display { + display.finish(result); + } } } -/// Wraps a dynamically dispatched tool without hiding its changing spec. -pub(crate) fn shared(tool: Arc) -> impl Tool { - Observed(SharedTool(tool)) +/// Wraps a dynamically dispatched tool without hiding specs or native outcomes. +pub(crate) fn shared(tool: Arc, observations: Observations) -> impl Tool { + Observed::new(SharedTool(tool)).with_observations(observations) } struct SharedTool(Arc); @@ -38,18 +64,15 @@ impl Tool for SharedTool { fn spec(&self) -> &ToolSpec { self.0.spec() } - fn current_spec(&self) -> Option { self.0.current_spec() } - fn proposed_requests( &self, request: &ToolRequest, ) -> Result>, ToolError> { self.0.proposed_requests(request) } - async fn invoke( &self, request: ToolRequest, @@ -57,32 +80,68 @@ impl Tool for SharedTool { ) -> Result { self.0.invoke(request, context).await } + async fn invoke_outcome( + &self, + request: ToolRequest, + context: &mut ToolContext<'_>, + ) -> ToolExecutionOutcome { + self.0.invoke_outcome(request, context).await + } } #[async_trait] impl Tool for Observed { fn spec(&self) -> &ToolSpec { - self.0.spec() + self.tool.spec() } - fn current_spec(&self) -> Option { - self.0.current_spec() + self.tool.current_spec() } - fn proposed_requests( &self, request: &ToolRequest, ) -> Result>, ToolError> { - self.0.proposed_requests(request) + self.tool.proposed_requests(request) } - async fn invoke( &self, request: ToolRequest, context: &mut ToolContext<'_>, ) -> Result { + let display = self.start(&request); + let outcome = self.tool.invoke(request, context).await; + self.finish(display, outcome.as_ref()); + outcome + } + async fn invoke_outcome( + &self, + request: ToolRequest, + context: &mut ToolContext<'_>, + ) -> ToolExecutionOutcome { + let display = self.start(&request); + let outcome = self.tool.invoke_outcome(request, context).await; + match &outcome { + ToolExecutionOutcome::Completed(result) => self.finish(display, Ok(result)), + ToolExecutionOutcome::Failed(error) + | ToolExecutionOutcome::FailedBeforeInvocation(error) => { + self.finish(display, Err(error)) + } + // Neither interruption nor dropping an in-flight future is completion. + ToolExecutionOutcome::Interrupted(_) => {} + } + outcome + } +} + +struct DisplayInvocation { + call: String, + tool: String, + started: Instant, +} +impl DisplayInvocation { + fn start(request: &ToolRequest) -> Option { if !events::enabled() { - return self.0.invoke(request, context).await; + return None; } let call = request.call_id.0.clone(); let tool = request.tool_name.0.to_string(); @@ -92,9 +151,14 @@ impl Tool for Observed { summary: summarize_input(&request.input), at: events::now_millis(), }); - let started = Instant::now(); - let outcome = self.0.invoke(request, context).await; - let (ok, summary) = match &outcome { + Some(Self { + call, + tool, + started: Instant::now(), + }) + } + fn finish(self, result: Result<&ToolResult, &ToolError>) { + let (ok, summary) = match result { Ok(result) => ( !result.result.is_error, summarize_output(&output_value(&result.result.output)), @@ -102,13 +166,12 @@ impl Tool for Observed { Err(error) => (false, summarize_output(&json!(error.to_string()))), }; events::emit(&RuntimeEvent::ChildFinished { - call, - tool, + call: self.call, + tool: self.tool, ok, summary, - millis: u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + millis: u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX), }); - outcome } } @@ -120,3 +183,171 @@ fn output_value(output: &ToolOutput) -> Value { ToolOutput::Files(files) => json!(format!("{} files", files.len())), } } + +#[cfg(test)] +mod tests { + use super::*; + use agentkit_core::{MetadataMap, SessionId, ToolCallId, ToolResultPart, TurnId}; + use agentkit_tools_core::{ + AllowAllPermissions, ApprovalReason, ApprovalRequest, OwnedToolContext, ToolInterruption, + ToolName, + }; + + #[derive(Clone, Copy)] + enum Mode { + Complete, + Pending, + Failed, + Cancelled, + Interrupted, + } + struct Fixture { + spec: ToolSpec, + mode: Mode, + } + impl Fixture { + fn new(mode: Mode) -> Self { + Self { + spec: ToolSpec::new(ToolName::new("fixture"), "fixture", json!({})), + mode, + } + } + } + #[async_trait] + impl Tool for Fixture { + fn spec(&self) -> &ToolSpec { + &self.spec + } + async fn invoke( + &self, + request: ToolRequest, + _: &mut ToolContext<'_>, + ) -> Result { + match self.mode { + Mode::Pending => std::future::pending().await, + Mode::Complete => Ok(ToolResult::new(ToolResultPart::success( + request.call_id, + ToolOutput::text("private result"), + ))), + Mode::Failed => Err(ToolError::ExecutionFailed("failed".into())), + Mode::Cancelled => Err(ToolError::Cancelled), + Mode::Interrupted => panic!("native outcome must not use invoke fallback"), + } + } + async fn invoke_outcome( + &self, + request: ToolRequest, + context: &mut ToolContext<'_>, + ) -> ToolExecutionOutcome { + if matches!(self.mode, Mode::Interrupted) { + return ToolExecutionOutcome::Interrupted(ToolInterruption::ApprovalRequired( + ApprovalRequest::new( + "approval", + "fixture", + ApprovalReason::PolicyRequiresConfirmation, + "approval", + ), + )); + } + match self.invoke(request, context).await { + Ok(result) => ToolExecutionOutcome::Completed(result), + Err(error) => ToolExecutionOutcome::Failed(error), + } + } + } + fn context() -> OwnedToolContext { + OwnedToolContext { + session_id: SessionId::new("session"), + turn_id: TurnId::new("turn"), + metadata: MetadataMap::new(), + permissions: Arc::new(AllowAllPermissions), + resources: Arc::new(()), + cancellation: None, + execution_scope: None, + approved_request: None, + } + } + fn request() -> ToolRequest { + ToolRequest::new( + ToolCallId::new("private-call"), + ToolName::new("fixture"), + json!({"secret": "private arguments"}), + SessionId::new("session"), + TurnId::new("turn"), + ) + } + + #[tokio::test] + async fn observations_do_not_depend_on_display_events() { + if crate::effects::isolated_test( + "tools::observed::tests::observations_do_not_depend_on_display_events", + ) { + return; + } + assert!(!events::enabled()); + let observations = Observations::local_session(); + let tool = + Observed::new(Fixture::new(Mode::Complete)).with_observations(observations.clone()); + tool.invoke(request(), &mut context().borrowed()) + .await + .unwrap(); + assert!(observations.snapshot().tool_execution_start_reported); + assert!(observations.snapshot().tool_execution_completion_reported); + assert!( + !serde_json::to_string(&observations.snapshot()) + .unwrap() + .contains("private") + ); + } + + #[tokio::test] + async fn dropping_running_invocation_does_not_report_completion() { + let observations = Observations::local_session(); + let tool = + Observed::new(Fixture::new(Mode::Pending)).with_observations(observations.clone()); + let owned = context(); + let mut context = owned.borrowed(); + let mut invocation = Box::pin(tool.invoke(request(), &mut context)); + tokio::select! { biased; _ = &mut invocation => panic!("pending"), () = tokio::task::yield_now() => {} } + drop(invocation); + assert!(observations.snapshot().tool_execution_start_reported); + assert!(!observations.snapshot().tool_execution_completion_reported); + assert!(observations.snapshot().observation_incomplete); + } + + #[tokio::test] + async fn shared_wrapper_preserves_native_interruption_failure_and_cancellation() { + for mode in [ + Mode::Complete, + Mode::Failed, + Mode::Cancelled, + Mode::Interrupted, + ] { + let observations = Observations::local_session(); + let tool = shared(Arc::new(Fixture::new(mode)), observations.clone()); + let outcome = tool + .invoke_outcome(request(), &mut context().borrowed()) + .await; + match mode { + Mode::Complete => assert!(matches!(outcome, ToolExecutionOutcome::Completed(_))), + Mode::Failed => assert!(matches!( + outcome, + ToolExecutionOutcome::Failed(ToolError::ExecutionFailed(_)) + )), + Mode::Cancelled => assert!(matches!( + outcome, + ToolExecutionOutcome::Failed(ToolError::Cancelled) + )), + Mode::Interrupted => { + assert!(matches!(outcome, ToolExecutionOutcome::Interrupted(_))) + } + Mode::Pending => unreachable!(), + } + assert!(observations.snapshot().tool_execution_start_reported); + assert_eq!( + observations.snapshot().tool_execution_completion_reported, + !matches!(mode, Mode::Interrupted) + ); + } + } +} diff --git a/src/tools/subagent.rs b/src/tools/subagent.rs index 2b56736b..8b280702 100644 --- a/src/tools/subagent.rs +++ b/src/tools/subagent.rs @@ -237,9 +237,9 @@ struct CreateOptions { cwd: Option, } -struct ForkSuccess { +struct ForkReply { effects: crate::effects::PossibleEffects, - value: SubagentValue, + value: Result, acknowledge: oneshot::Sender<()>, } @@ -551,8 +551,10 @@ impl Subagents { } } + #[allow(clippy::too_many_arguments)] async fn fork( &self, + parent_session_id: String, prior: SubagentValue, prompt: String, name: Option, @@ -619,33 +621,29 @@ impl Subagents { let reservation = operation.id.clone(); let result = manager.run_fork(operation, &reply).await; manager.finish_forking(&source_state, &reservation).await; - match result { - Ok((value, effects)) => manager.handoff_fork_success(reply, value, effects).await, - Err(error) => { - let _ = reply.send(Err(error)); - } - } + manager + .handoff_fork_result(&parent_session_id, reply, result) + .await; }); - match response.await.map_err(|_| { + let response = response.await.map_err(|_| { ChildError::Failed("subagent fork task stopped before returning a result".into()) - })? { - Ok(success) => { - success.acknowledge.send(()).map_err(|_| { - ChildError::Failed( - "subagent fork task stopped before transferring ownership".into(), - ) - .with_effects(success.effects) - })?; - Ok(success.value) - } - Err(error) => Err(error), + })?; + if response.acknowledge.send(()).is_err() { + return Err(match response.value { + Err(error) => error, + Ok(_) => ChildError::Failed( + "subagent fork task stopped before transferring ownership".into(), + ) + .with_effects(response.effects), + }); } + response.value } async fn run_fork( &self, operation: ForkOperation, - reply: &oneshot::Sender>, + reply: &oneshot::Sender, ) -> Result<(SubagentValue, crate::effects::PossibleEffects), ChildError> { let ForkOperation { source_id, @@ -985,23 +983,32 @@ impl Subagents { }) } - async fn handoff_fork_success( + async fn handoff_fork_result( &self, - reply: oneshot::Sender>, - value: SubagentValue, - effects: crate::effects::PossibleEffects, + parent_session_id: &str, + reply: oneshot::Sender, + result: Result<(SubagentValue, crate::effects::PossibleEffects), ChildError>, ) { - let cleanup = value.clone(); + let (cleanup, effects) = match &result { + Ok((value, effects)) => (Some(value.clone()), *effects), + Err(error) => (None, error.possible_effects()), + }; let (acknowledge, acknowledged) = oneshot::channel(); let sent = reply - .send(Ok(ForkSuccess { - value, + .send(ForkReply { + value: result.map(|(value, _)| value), effects, acknowledge, - })) + }) .is_ok(); + // Both outcomes need acknowledgment: send can succeed while the caller + // drops its future before receiving the reply. Only an acknowledged + // caller owns failure recording through result(). if !sent || acknowledged.await.is_err() { - self.cleanup_abandoned_fork(&cleanup).await; + if let Some(value) = cleanup { + self.cleanup_abandoned_fork(&value).await; + } + record_child_failure(parent_session_id, effects); } } @@ -1492,15 +1499,18 @@ fn tool_failure(error: &ChildError) -> ToolError { } } +fn record_child_failure(session_id: &str, effects: crate::effects::PossibleEffects) { + if let Err(log_error) = crate::fatal::record_child_failure(session_id, effects) { + tracing::warn!(%log_error, "could not store child failure observations"); + } +} + fn result( request: ToolRequest, value: Result, ) -> Result { let value = value.map_err(|error| { - let effects = error.possible_effects(); - if let Err(log_error) = crate::fatal::record_child_failure(&request.session_id.0, effects) { - tracing::warn!(%log_error, "could not store child failure observations"); - } + record_child_failure(&request.session_id.0, error.possible_effects()); tool_failure(&error) })?; Ok(ToolResult::new(ToolResultPart::success( @@ -1640,10 +1650,12 @@ impl Tool for ForkTool { let input: ForkInput = serde_json::from_value(request.input.clone()) .map_err(|e| ToolError::InvalidInput(e.to_string()))?; let contract = input.output_schema.map(OutputContract::new).transpose()?; + let parent_session_id = request.session_id.0.clone(); result( request, self.manager .fork( + parent_session_id, input.subagent, input.prompt, input.name, diff --git a/src/tools/subagent/tests.rs b/src/tools/subagent/tests.rs index da44321e..577582ac 100644 --- a/src/tools/subagent/tests.rs +++ b/src/tools/subagent/tests.rs @@ -625,6 +625,7 @@ async fn create_uses_requested_working_directory_without_changing_parent() { let branch = manager .fork( + session::new_id(), source.clone(), "MOCK_CWD".into(), None, @@ -844,6 +845,7 @@ impl MockAcpScenario { tokio::spawn(async move { manager .fork( + session::new_id(), source, prompt.into(), None, @@ -1197,6 +1199,7 @@ async fn failed_create_and_fork_startup_record_failed_removed_transitions() { assert!( failed_fork .fork( + session::new_id(), source, "fork".into(), None, @@ -1350,6 +1353,7 @@ async fn native_fork_releases_the_source_before_the_branch_prompt() { let fork_error = scenario .manager .fork( + session::new_id(), source.clone(), "second branch".into(), None, @@ -1405,19 +1409,40 @@ async fn native_fork_releases_the_source_before_the_branch_prompt() { #[tokio::test] async fn dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit() { + if crate::effects::isolated_test( + "tools::subagent::tests::dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit", + ) { + return; + } let scenario = MockAcpScenario::new(ScenarioOptions { - gate_prompt: Some("branch"), + gate_prompt: Some("MOCK_RICH_OUTPUT"), fail_close_session: Some("branch-1"), ..Default::default() }); let source = scenario.create("source").await; - let fork = scenario.spawn_fork(source.clone(), "branch"); + let parent_session_id = session::new_id(); + let fork_manager = scenario.manager.clone(); + let fork_source = source.clone(); + let parent = parent_session_id.clone(); + let fork = tokio::spawn(async move { + fork_manager + .fork( + parent, + fork_source, + "MOCK_RICH_OUTPUT".into(), + None, + 0, + TurnCancellation::default(), + None, + ) + .await + }); scenario .wait_for(|request| { matches!( request, LoggedRequest::Prompt { session_id, text } - if session_id == "branch-1" && text == "branch" + if session_id == "branch-1" && text == "MOCK_RICH_OUTPUT" ) }) .await; @@ -1456,6 +1481,9 @@ async fn dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit [source.id.as_str()] ); + wait_for_fork_diagnostic(&parent_session_id).await; + assert_rich_fork_diagnostic(&parent_session_id); + scenario .manager .close(&source.id, &TurnCancellation::default()) @@ -1464,32 +1492,215 @@ async fn dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; } -#[tokio::test] -async fn successful_fork_handoff_cleans_up_if_receipt_is_not_acknowledged() { - let scenario = MockAcpScenario::new(ScenarioOptions::default()); - let branch = scenario.create("branch").await; - let (reply, response) = oneshot::channel(); - let manager = scenario.manager.clone(); - let cleanup_branch = branch.clone(); - let handoff = tokio::spawn(async move { - manager - .handoff_fork_success(reply, cleanup_branch, Default::default()) - .await; - }); +fn fork_diagnostic_count(session_id: &str) -> usize { + let directory = PathBuf::from(std::env::var_os("HOME").unwrap()) + .join(".kit/errors") + .join(session_id); + std::fs::read_dir(directory) + .into_iter() + .flatten() + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .count() +} - let success = response.await.unwrap().unwrap(); - assert_eq!(success.value.id, branch.id); - drop(success); - handoff.await.unwrap(); +async fn wait_for_fork_diagnostic(session_id: &str) { + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while fork_diagnostic_count(session_id) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("detached fork did not retain observations"); +} - assert!( - scenario - .manager - .list(&TurnCancellation::default()) - .await - .unwrap() - .is_empty() +fn assert_rich_fork_diagnostic(session_id: &str) { + assert_eq!(fork_diagnostic_count(session_id), 1); + let record = crate::effects::test_record(session_id); + assert_eq!(record["session_id"], session_id); + assert_eq!(record["surface"], "subagent"); + assert_eq!(record["code"], "subagent_failed"); + assert_eq!( + record["possible_effects"], + json!({ + "source": "acp_notifications", + "assistant_output_observed": true, + "tool_emission_observed": true, + "tool_execution_start_reported": false, + "tool_execution_completion_reported": true, + "observation_incomplete": true, + }) + ); + let encoded = record.to_string(); + for private in ["rich done", "call-1", "Inspect files", "MOCK_RICH_OUTPUT"] { + assert!(!encoded.contains(private)); + } +} + +#[tokio::test] +async fn fork_handoff_retains_observations_only_when_caller_abandons_delivery() { + if crate::effects::isolated_test( + "tools::subagent::tests::fork_handoff_retains_observations_only_when_caller_abandons_delivery", + ) { + return; + } + // Exercise closed delivery, dropped acknowledgment, and normal delivery for + // both outcomes. Positive facts come from actual ACP notifications first. + for failed in [false, true] { + for delivery in ["closed", "unacknowledged", "acknowledged"] { + let scenario = MockAcpScenario::new(ScenarioOptions::default()); + let branch = scenario.create("branch").await; + let state = scenario.manager.lookup(&branch).unwrap(); + let child = state.lock().await.child.clone().unwrap(); + let effects = child + .prompt("MOCK_RICH_OUTPUT".into(), TurnCancellation::default()) + .await + .unwrap() + .possible_effects(); + assert!(effects.tool_execution_completion_reported); + let outcome = if failed { + Err(scenario + .manager + .cleanup_installed_child( + &branch.id, + &state, + &child, + ChildError::Cancelled.with_effects(effects), + ) + .await) + } else { + Ok((branch.clone(), effects)) + }; + let parent_session_id = session::new_id(); + let parent = parent_session_id.clone(); + let (reply, response) = oneshot::channel(); + let manager = scenario.manager.clone(); + let response = if delivery == "closed" { + drop(response); + None + } else { + Some(response) + }; + let handoff = tokio::spawn(async move { + manager.handoff_fork_result(&parent, reply, outcome).await; + }); + let mut delivered = None; + if let Some(response) = response { + let receipt = response.await.unwrap(); + assert_eq!(receipt.effects, effects); + if delivery == "acknowledged" { + receipt.acknowledge.send(()).unwrap(); + delivered = Some(receipt.value); + } + } + handoff.await.unwrap(); + if let Some(value) = delivered { + // The detached owner must not duplicate the normal result writer. + assert_eq!(fork_diagnostic_count(&parent_session_id), 0); + let request = ToolRequest::new( + agentkit_core::ToolCallId::new("fork-call"), + ToolName::new("fork"), + json!({}), + agentkit_core::SessionId::new(parent_session_id.clone()), + agentkit_core::TurnId::new("fork-turn"), + ); + let result = result(request, value); + if failed { + assert!(matches!(result, Err(ToolError::Cancelled))); + assert_rich_fork_diagnostic(&parent_session_id); + } else { + assert!(result.is_ok()); + assert_eq!(fork_diagnostic_count(&parent_session_id), 0); + scenario + .manager + .close(&branch.id, &TurnCancellation::default()) + .await + .unwrap(); + } + } else { + assert_rich_fork_diagnostic(&parent_session_id); + } + assert!( + scenario + .manager + .list(&TurnCancellation::default()) + .await + .unwrap() + .is_empty() + ); + // The fixture's extra State reference still owns the process permit. + drop(child); + drop(state); + wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; + } + } +} + +#[tokio::test] +async fn dropped_fork_after_positive_observations_retains_diagnostic() { + if crate::effects::isolated_test( + "tools::subagent::tests::dropped_fork_after_positive_observations_retains_diagnostic", + ) { + return; + } + let mut scenario = MockAcpScenario::new(ScenarioOptions { + gate_prompt: Some("MOCK_RICH_OUTPUT"), + ..Default::default() + }); + let (manager, events) = observe_events(scenario.manager.clone()); + scenario.manager = manager; + let source = scenario.create("source").await; + let parent_session_id = session::new_id(); + let mut fork = Box::pin(scenario.manager.fork( + parent_session_id.clone(), + source.clone(), + "MOCK_RICH_OUTPUT".into(), + None, + 0, + TurnCancellation::default(), + None, + )); + tokio::select! { + _ = &mut fork => panic!("gated fork returned early"), + _ = scenario.wait_for(|request| matches!(request, + LoggedRequest::Prompt { text, .. } if text == "MOCK_RICH_OUTPUT")) => {} + } + MockAcpScenario::release(&scenario.prompt_release); + // Leave the caller unpolled until run_fork has observed the child output + // and published success. Its queued reply has not been acknowledged. + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if emitted(&events).iter().any(|event| { + matches!(event, + events::RuntimeEvent::SubagentStateChanged { + id, status: SubagentStatus::Idle, .. + } if id != &source.id) + }) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("fork did not observe successful child output"); + assert_eq!(fork_diagnostic_count(&parent_session_id), 0); + drop(fork); + wait_for_fork_diagnostic(&parent_session_id).await; + assert_rich_fork_diagnostic(&parent_session_id); + let listed = scenario + .manager + .list(&TurnCancellation::default()) + .await + .unwrap(); + assert_eq!( + listed.iter().map(listing_id).collect::>(), + [source.id.as_str()] ); + scenario + .manager + .close(&source.id, &TurnCancellation::default()) + .await + .unwrap(); wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; } @@ -1782,6 +1993,7 @@ async fn fork_uses_its_fresh_preferred_name() { let fork = manager .fork( + session::new_id(), source, "branch".into(), Some("Reviewer".into()), @@ -1842,6 +2054,7 @@ async fn generic_harness_without_native_fork_returns_unsupported() { let error = manager .fork( + session::new_id(), prior, "branch".into(), None, diff --git a/tests/runtime.rs b/tests/runtime.rs index d5b1b9e0..f12f433f 100644 --- a/tests/runtime.rs +++ b/tests/runtime.rs @@ -1,6 +1,9 @@ use std::{ collections::BTreeMap, - sync::Arc, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, time::{Duration, Instant}, }; @@ -626,6 +629,13 @@ async fn execute_compose_cancelled( script: &str, cancellation: Option, ) -> ToolExecutionOutcome { + // Artifact paths are HOME/session/call scoped. Parallel tests must not + // remove a spill directory still in use by another invocation. + static NEXT_CALL: AtomicUsize = AtomicUsize::new(0); + let call_id = ToolCallId::new(format!( + "compose-test-{}", + NEXT_CALL.fetch_add(1, Ordering::Relaxed) + )); let source: Arc = Arc::new(runtime.compose(0)); let executor = Arc::new(BasicToolExecutor::new([source])); let scope = ToolExecutionScope { @@ -638,7 +648,7 @@ async fn execute_compose_cancelled( }; scope .execute_child(ToolRequest { - call_id: ToolCallId::new("compose-test"), + call_id, tool_name: ToolName::new("compose"), input: json!({"script": script}), session_id: SessionId::new("test"), From a61fa9f64c0015db5ea45b945f62de8b0da7265b Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 21:51:55 +0100 Subject: [PATCH 3/7] fix: address effects provenance review feedback --- docs/issues/possible-effects.md | 71 -------------- docs/user/tui-and-sessions.md | 2 +- src/fatal.rs | 165 ++++++++++++++++++++++++-------- src/tools/subagent.rs | 29 ++++-- src/tools/subagent/tests.rs | 80 +++++++++------- 5 files changed, 192 insertions(+), 155 deletions(-) delete mode 100644 docs/issues/possible-effects.md diff --git a/docs/issues/possible-effects.md b/docs/issues/possible-effects.md deleted file mode 100644 index fe409ad2..00000000 --- a/docs/issues/possible-effects.md +++ /dev/null @@ -1,71 +0,0 @@ -# Possible-effects observations (issue #48) - -A failed prompt can have changed external state. Kit retains bounded, -provider-independent positive observations independently of output-retention -limits and opt-in stderr display events. - -`possible_effects` contains only fixed field names, an allowlisted source, and -booleans. It never contains tool IDs, arguments, results, assistant text, prompts, -or provider payloads. Sources distinguish two observation scopes: - -- `acp_notifications`: reports received during one dispatched child prompt. - Pending tool announcements are not starts. A completed status does not imply - that a start was observed. A failed status can reflect pre-execution denial, - so it does not establish completed execution. -- `local_session`: cumulative observations during one **live root owner's - lifetime**, shared by its loop observer, compose, and hidden tool wrappers. - These are not facts attributable exclusively to the failing prompt, nor do - they cover earlier process instances or a resumed session's stored history. - Execution starts and completions come from actual invocation entry/terminal - return boundaries, not synthesized tool-result events. Entry into a local - invocation does not establish dispatch or execution of an external operation. -- `unknown`: no explicit observation source is available. - -Local assistant-output tracking excludes reasoning and tool-argument deltas, -handles committed content without prior deltas, and bounds transient part -classification by identifier length and entry count. Presentation supersession -and new prompts clear only transient classification, never positive evidence. -Detached work retains the same owner when it spans prompts. - -All failure snapshots remain `observation_incomplete: true`. False means -**not observed**, not **did not happen**. A completion boolean means some -invocation completed; it does not establish that every invocation or external -operation completed. Interruptions and dropped futures are not completions. -Completion establishes neither successful external effects, rollback, nor replay -safety. No automatic retry or reconstruction decision follows from these facts. - -Child observation ownership survives dispatched-task/channel loss and -post-response ownership rejection. Cleanup failures preserve the original -failure classification. Root finalization may return a cleanup or flush error instead -of the original driver error; this gets a separate `session_finalization` -diagnostic without replacing the earlier driver record. Root prompt, ACP v1/v2, and A2A failure paths retain -explicit owners, including unstructured and unsolicited execution. Cancellation -keeps its existing lifecycle classification and records a snapshot after -structured cleanup when available; Kit does not wait for unrelated detached work -merely to improve evidence. - -Fatal writers use schema version 4, adding the `local_session` source to version -3's effects object. New readers retain v1/v2 missing-effects defaults and v3 -`unknown`/`acp_notifications` records; existing supplied effects are preserved. -Malformed fields, false completeness claims, and unknown future source values are rejected. This backward -read compatibility does not mean an older strict reader understands a new -source value. Files are not rewritten on read. Parent-side observation records -are distinct diagnostics, not recreated child fatal receipt identities. - -## Remaining typed transport and release gates - -The pinned AgentKit `ToolError` exposes string execution failures and unit -cancellation. The existing parent error behavior remains unchanged. The shared -upstream API implementation is authorized and reliability-owned; Kit integration -waits for reviewed compatible published crates rather than Cargo patches. - -Required integration is typed, validated metadata through ACP error data, child -failures, subagent tools, Runlet's documented catch/rethrow policy, and both -foreground/background terminal projections, retaining one child fatal receipt -with explicit storage disposition. Cancellation must keep its classification. -A rendered JSON string is not a substitute for that contract. - -Do not return a completed `ToolResult` with `is_error: true` as a workaround: -the pinned Compose dispatcher treats completed child results as successful -values, bypassing Runlet error boundaries. Recovery (#21) remains gated until -complete #48 delivery and review; effects alone cannot reconstruct a session. diff --git a/docs/user/tui-and-sessions.md b/docs/user/tui-and-sessions.md index 2eb3ab3a..6f6ecf54 100644 --- a/docs/user/tui-and-sessions.md +++ b/docs/user/tui-and-sessions.md @@ -150,7 +150,7 @@ Kit stores durable JSONL transcripts, locks, and session-associated fatal error The workspace hash is the BLAKE3 digest of the canonical workspace-root path. It keeps identical session IDs in different workspaces in separate storage directories. The optional metadata sidecar stores only the custom display name, is replaced atomically, and does not modify or lock the append-only transcript. Missing or malformed metadata falls back to the generated title without hiding the session. -Fatal error records use their own versioned JSON schema and are not transcript content. Schema v2 adds optional structured transport diagnostics; schema v1 records remain readable. Transport diagnostics contain only bounded, allowlisted request/stream stage, retry, attempt, the provider's strictly validated `x-request-id` value, reqwest classification, and typed Hyper, HTTP/2, and I/O fields. Unknown or truncated source chains are identified without storing source text. Kit never stores raw error display/debug text, arbitrary headers, prompts, tool arguments, response bodies, credentials, URLs, or peer-controlled HTTP/2 debug text in these records. Files are written atomically with owner-only permissions on Unix, and Kit retains the newest 50 records per session. Cancellation is not a fatal error and does not create a record. When persistence succeeds, local prompt and ACP terminal errors include the log path; A2A records stay server-local. +Fatal error records use their own versioned JSON schema and are not transcript content. Schema v2 supports optional structured transport diagnostics and additive possible-effects observations; schema v1 records remain readable. Missing observations mean unknown activity, not absence of effects. Observations are incomplete and never establish that retrying is safe. Transport diagnostics contain only bounded, allowlisted request/stream stage, retry, attempt, the provider's strictly validated `x-request-id` value, reqwest classification, and typed Hyper, HTTP/2, and I/O fields. Unknown or truncated source chains are identified without storing source text. Kit never stores raw error display/debug text, arbitrary headers, prompts, tool arguments, response bodies, credentials, URLs, or peer-controlled HTTP/2 debug text in these records. Files are written atomically with owner-only permissions on Unix, and Kit retains the newest 50 records per session. Cancellation remains a cancellation outcome, but can create a diagnostic record to retain possible-effects observations. When persistence succeeds, local prompt and ACP terminal errors include the log path; A2A records stay server-local. `HOME is unset; cannot locate durable sessions` means Kit cannot determine this directory. Set `HOME` to the intended home directory before starting Kit. diff --git a/src/fatal.rs b/src/fatal.rs index 5585ffed..1b5e74d3 100644 --- a/src/fatal.rs +++ b/src/fatal.rs @@ -9,7 +9,7 @@ use agentkit_loop::LoopError; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use serde::{Deserialize, Serialize}; -const SCHEMA_VERSION: u64 = 4; +const SCHEMA_VERSION: u64 = 2; const MAX_MESSAGE_BYTES: usize = 2 * 1024; const MAX_RECORD_BYTES: usize = 32 * 1024; const MAX_RECORDS_PER_SESSION: usize = 50; @@ -428,9 +428,11 @@ fn write_default_with_effects( &PathBuf::from(home).join(".kit/errors"), session_id, surface, - kind, - code, - message, + FatalDetails { + kind, + code, + message, + }, diagnostics, effects, ) @@ -462,9 +464,11 @@ fn write_in_with_diagnostics( base, session_id, surface, - kind, - code, - message, + FatalDetails { + kind, + code, + message, + }, diagnostics, crate::effects::PossibleEffects::default(), ) @@ -474,11 +478,7 @@ pub(crate) fn record_child_failure( session_id: &str, effects: crate::effects::PossibleEffects, ) -> Result { - let home = std::env::var_os("HOME") - .filter(|home| !home.is_empty()) - .ok_or_else(|| "HOME is unset; cannot store fatal error log".to_owned())?; - write_in_with_effects( - &PathBuf::from(home).join(".kit/errors"), + write_default_with_effects( session_id, Surface::Subagent, "runtime", @@ -489,14 +489,17 @@ pub(crate) fn record_child_failure( ) } -#[allow(clippy::too_many_arguments)] +struct FatalDetails<'a> { + kind: &'a str, + code: &'a str, + message: &'a str, +} + fn write_in_with_effects( base: &Path, session_id: &str, surface: Surface, - kind: &str, - code: &str, - message: &str, + details: FatalDetails<'_>, diagnostics: Option<&TransportDiagnostics>, possible_effects: crate::effects::PossibleEffects, ) -> Result { @@ -517,9 +520,9 @@ fn write_in_with_effects( kit_version: env!("CARGO_PKG_VERSION").into(), session_id: session_id.into(), surface: surface.as_str().into(), - kind: kind.into(), - code: canonical_code(code).into(), - message: bounded(message), + kind: details.kind.into(), + code: canonical_code(details.code).into(), + message: bounded(details.message), diagnostics: diagnostics.filter(|value| value.valid()).cloned(), possible_effects, }; @@ -678,7 +681,7 @@ mod tests { .unwrap(); assert_eq!(path.parent().unwrap(), root.path().join("session-1")); let record: FatalRecord = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - assert_eq!(record.schema_version, 4); + assert_eq!(record.schema_version, 2); assert_eq!(record.session_id, "session-1"); assert_eq!(record.surface, "prompt"); assert_eq!(record.code, "stream_transport"); @@ -696,9 +699,11 @@ mod tests { root.path(), "session-effects", Surface::Subagent, - "runtime", - "subagent_failed", - "nested agent failed", + super::FatalDetails { + kind: "runtime", + code: "subagent_failed", + message: "nested agent failed", + }, None, effects, ) @@ -707,17 +712,77 @@ mod tests { serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); let current: FatalRecord = serde_json::from_value(value.clone()).unwrap(); assert_eq!(current.possible_effects, effects); - value["schema_version"] = json!(2); - value.as_object_mut().unwrap().remove("possible_effects"); - let legacy: FatalRecord = serde_json::from_value(value.clone()).unwrap(); - assert_eq!( - legacy.possible_effects, - crate::effects::PossibleEffects::default() - ); - // An additive field on an older version must not be overwritten. - value["possible_effects"] = serde_json::to_value(effects).unwrap(); - let mixed: FatalRecord = serde_json::from_value(value).unwrap(); - assert_eq!(mixed.possible_effects, effects); + assert_eq!(current.schema_version, 2); + for marker in [1, 2, 3, 4] { + value["schema_version"] = json!(marker); + value.as_object_mut().unwrap().remove("possible_effects"); + let legacy: FatalRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + legacy.possible_effects, + crate::effects::PossibleEffects::default() + ); + // Presence, not the numeric marker, determines whether observations exist. + value["possible_effects"] = serde_json::to_value(effects).unwrap(); + let extended: FatalRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(extended.possible_effects, effects); + } + } + + #[test] + fn frozen_v2_reader_retains_known_fields_from_extended_records() { + // Pre-effects top-level reader; the transport diagnostics shape is unchanged. + #[derive(Debug, serde::Deserialize, serde::Serialize)] + struct FatalRecordV2 { + schema_version: u64, + event_id: String, + occurred_at_ms: u64, + kit_version: String, + session_id: String, + surface: String, + kind: String, + code: String, + message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + diagnostics: Option, + } + + let root = tempfile::tempdir().unwrap(); + let diagnostics = sample_diagnostics(); + for source in [ + crate::effects::ObservationSource::Unknown, + crate::effects::ObservationSource::AcpNotifications, + crate::effects::ObservationSource::LocalSession, + ] { + let effects = crate::effects::PossibleEffects { + source, + tool_execution_completion_reported: true, + ..Default::default() + }; + let path = super::write_in_with_effects( + root.path(), + "session-extended", + Surface::Subagent, + super::FatalDetails { + kind: "runtime", + code: "subagent_failed", + message: "nested agent failed", + }, + Some(&diagnostics), + effects, + ) + .unwrap(); + let bytes = fs::read(path).unwrap(); + let legacy: FatalRecordV2 = serde_json::from_slice(&bytes).unwrap(); + let current: FatalRecord = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(legacy.schema_version, 2); + assert_eq!(current.possible_effects, effects); + let mut known_fields: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + known_fields + .as_object_mut() + .unwrap() + .remove("possible_effects"); + assert_eq!(serde_json::to_value(legacy).unwrap(), known_fields); + } } #[test] @@ -756,7 +821,7 @@ mod tests { let encoded = fs::read_to_string(path).unwrap(); let value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); - assert_eq!(value["schema_version"], 4); + assert_eq!(value["schema_version"], 2); assert_eq!(value["diagnostics"]["response_request_id"], "req_safe-123"); assert_eq!(value["diagnostics"]["stage"], "stream"); assert_eq!(value["diagnostics"]["retryable"], true); @@ -982,7 +1047,7 @@ mod tests { .unwrap() .unwrap(); let value: serde_json::Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - assert_eq!(value["schema_version"], 4); + assert_eq!(value["schema_version"], 2); assert_eq!(value["kind"], "cancelled"); assert_eq!(value["code"], "cancelled"); assert_eq!(value["possible_effects"]["source"], "local_session"); @@ -994,7 +1059,7 @@ mod tests { } #[test] - fn schema_v3_effects_and_new_local_source_remain_strict() { + fn supplied_effects_remain_strict() { let root = tempfile::tempdir().unwrap(); let path = write_in( root.path(), @@ -1007,7 +1072,6 @@ mod tests { .unwrap(); let mut value: serde_json::Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - value["schema_version"] = json!(3); for source in ["unknown", "acp_notifications", "local_session"] { value["possible_effects"]["source"] = json!(source); let record: FatalRecord = serde_json::from_value(value.clone()).unwrap(); @@ -1016,9 +1080,26 @@ mod tests { source ); } - value["possible_effects"]["source"] = json!("unknown-future-source"); - assert!(serde_json::from_value::(value.clone()).is_err()); - value["possible_effects"] = json!({"source": "local_session", "private_payload": "secret"}); - assert!(serde_json::from_value::(value).is_err()); + for (field, invalid) in [ + ("source", json!("unknown-future-source")), + ("assistant_output_observed", json!("true")), + ("observation_incomplete", json!(false)), + ("private_payload", json!("secret")), + ] { + let mut malformed = value.clone(); + malformed["possible_effects"][field] = invalid; + assert!( + serde_json::from_value::(malformed).is_err(), + "{field}" + ); + } + for invalid in [ + serde_json::Value::Null, + json!({"source": "local_session"}), + json!([]), + ] { + value["possible_effects"] = invalid; + assert!(serde_json::from_value::(value.clone()).is_err()); + } } } diff --git a/src/tools/subagent.rs b/src/tools/subagent.rs index 8b280702..99103675 100644 --- a/src/tools/subagent.rs +++ b/src/tools/subagent.rs @@ -237,6 +237,13 @@ struct CreateOptions { cwd: Option, } +struct ForkRequest { + prior: SubagentValue, + prompt: String, + name: Option, + contract: Option>, +} + struct ForkReply { effects: crate::effects::PossibleEffects, value: Result, @@ -551,17 +558,19 @@ impl Subagents { } } - #[allow(clippy::too_many_arguments)] async fn fork( &self, parent_session_id: String, - prior: SubagentValue, - prompt: String, - name: Option, + request: ForkRequest, depth: usize, cancellation: TurnCancellation, - contract: Option>, ) -> Result { + let ForkRequest { + prior, + prompt, + name, + contract, + } = request; self.check_depth(depth)?; let permit = self.reserve()?; let source_state = self.lookup(&prior)?; @@ -1656,12 +1665,14 @@ impl Tool for ForkTool { self.manager .fork( parent_session_id, - input.subagent, - input.prompt, - input.name, + ForkRequest { + prior: input.subagent, + prompt: input.prompt, + name: input.name, + contract: contract.map(Arc::new), + }, self.depth, cancellation(context), - contract.map(Arc::new), ) .await, ) diff --git a/src/tools/subagent/tests.rs b/src/tools/subagent/tests.rs index 577582ac..5f924bf1 100644 --- a/src/tools/subagent/tests.rs +++ b/src/tools/subagent/tests.rs @@ -626,12 +626,14 @@ async fn create_uses_requested_working_directory_without_changing_parent() { let branch = manager .fork( session::new_id(), - source.clone(), - "MOCK_CWD".into(), - None, + super::ForkRequest { + prior: source.clone(), + prompt: "MOCK_CWD".into(), + name: None, + contract: None, + }, 0, TurnCancellation::default(), - None, ) .await .unwrap(); @@ -846,12 +848,14 @@ impl MockAcpScenario { manager .fork( session::new_id(), - source, - prompt.into(), - None, + super::ForkRequest { + prior: source, + prompt: prompt.into(), + name: None, + contract: None, + }, 0, TurnCancellation::default(), - None, ) .await }) @@ -1200,12 +1204,14 @@ async fn failed_create_and_fork_startup_record_failed_removed_transitions() { failed_fork .fork( session::new_id(), - source, - "fork".into(), - None, + super::ForkRequest { + prior: source, + prompt: "fork".into(), + name: None, + contract: None, + }, 0, TurnCancellation::default(), - None, ) .await .is_err() @@ -1354,12 +1360,14 @@ async fn native_fork_releases_the_source_before_the_branch_prompt() { .manager .fork( session::new_id(), - source.clone(), - "second branch".into(), - None, + super::ForkRequest { + prior: source.clone(), + prompt: "second branch".into(), + name: None, + contract: None, + }, 0, TurnCancellation::default(), - None, ) .await .unwrap_err(); @@ -1428,12 +1436,14 @@ async fn dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit fork_manager .fork( parent, - fork_source, - "MOCK_RICH_OUTPUT".into(), - None, + super::ForkRequest { + prior: fork_source, + prompt: "MOCK_RICH_OUTPUT".into(), + name: None, + contract: None, + }, 0, TurnCancellation::default(), - None, ) .await }); @@ -1653,12 +1663,14 @@ async fn dropped_fork_after_positive_observations_retains_diagnostic() { let parent_session_id = session::new_id(); let mut fork = Box::pin(scenario.manager.fork( parent_session_id.clone(), - source.clone(), - "MOCK_RICH_OUTPUT".into(), - None, + super::ForkRequest { + prior: source.clone(), + prompt: "MOCK_RICH_OUTPUT".into(), + name: None, + contract: None, + }, 0, TurnCancellation::default(), - None, )); tokio::select! { _ = &mut fork => panic!("gated fork returned early"), @@ -1994,12 +2006,14 @@ async fn fork_uses_its_fresh_preferred_name() { let fork = manager .fork( session::new_id(), - source, - "branch".into(), - Some("Reviewer".into()), + super::ForkRequest { + prior: source, + prompt: "branch".into(), + name: Some("Reviewer".into()), + contract: None, + }, 0, TurnCancellation::default(), - None, ) .await .unwrap(); @@ -2055,12 +2069,14 @@ async fn generic_harness_without_native_fork_returns_unsupported() { let error = manager .fork( session::new_id(), - prior, - "branch".into(), - None, + super::ForkRequest { + prior: prior, + prompt: "branch".into(), + name: None, + contract: None, + }, 0, TurnCancellation::default(), - None, ) .await .unwrap_err(); From 81b24fcb6b4146fe5b18aac9278e95bb0e3e9091 Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 21:54:55 +0100 Subject: [PATCH 4/7] fix: use shorthand in fork request fixture --- src/tools/subagent/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/subagent/tests.rs b/src/tools/subagent/tests.rs index 5f924bf1..abfa8324 100644 --- a/src/tools/subagent/tests.rs +++ b/src/tools/subagent/tests.rs @@ -2070,7 +2070,7 @@ async fn generic_harness_without_native_fork_returns_unsupported() { .fork( session::new_id(), super::ForkRequest { - prior: prior, + prior, prompt: "branch".into(), name: None, contract: None, From 04c733541684cfd0f73e7d744b19772436ff5dcc Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 22:01:43 +0100 Subject: [PATCH 5/7] fix(acp): bundle autonomous session instruments --- src/protocols/acp/v2.rs | 68 ++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/src/protocols/acp/v2.rs b/src/protocols/acp/v2.rs index db6e3c7e..563f9a32 100644 --- a/src/protocols/acp/v2.rs +++ b/src/protocols/acp/v2.rs @@ -1210,7 +1210,11 @@ async fn session_actor(actor: SessionActor) &busy, &mut driver, &sink, - &activity, &background_jobs.observations,).await, + SessionInstruments { + activity: &activity, + observations: &background_jobs.observations, + }, + ).await, Err(error) => Err(map_loop_error_observed(&session_id, &error, &background_jobs.observations)), }; if let Err(error) = result { @@ -1229,7 +1233,11 @@ async fn session_actor(actor: SessionActor) &busy, &mut driver, &sink, - &activity, &background_jobs.observations,).await + SessionInstruments { + activity: &activity, + observations: &background_jobs.observations, + }, + ).await { eprintln!("ACP v2 autonomous turn failed for {session_id}: {error}"); } @@ -1648,7 +1656,12 @@ async fn run_active_turn( .map(|_| ()) } -#[allow(clippy::too_many_arguments)] +/// Session-owned lifecycle and effects instruments used during execution. +struct SessionInstruments<'a> { + activity: &'a SessionActivity, + observations: &'a crate::effects::Observations, +} + async fn drive_autonomous( session_id: &wire::SessionId, integration: &AcpIntegration, @@ -1656,8 +1669,7 @@ async fn drive_autonomous( busy: &AtomicBool, driver: &mut LoopDriver, sink: &ResponseReplacementSink, - activity: &SessionActivity, - observations: &crate::effects::Observations, + instruments: SessionInstruments<'_>, ) -> Result<(), AcpRuntimeError> { if claim_prompt(busy).is_err() { return Ok(()); @@ -1674,9 +1686,9 @@ async fn drive_autonomous( sink, cancellation_generation, None, - activity, + instruments.activity, ExecutionOrigin::Autonomous, - observations, + instruments.observations, ) .await; integration.finish_prompt(session_id); @@ -3562,8 +3574,10 @@ mod tests { &busy, &mut driver, &sink, - &activity, - &crate::effects::Observations::local_session(), + SessionInstruments { + activity: &activity, + observations: &crate::effects::Observations::local_session(), + }, ) .await .unwrap(); @@ -3620,8 +3634,10 @@ mod tests { &busy, &mut driver, &sink, - &activity, - &crate::effects::Observations::local_session(), + SessionInstruments { + activity: &activity, + observations: &crate::effects::Observations::local_session(), + }, ) .await .unwrap(); @@ -3638,8 +3654,10 @@ mod tests { &busy, &mut driver, &sink, - &activity, - &crate::effects::Observations::local_session(), + SessionInstruments { + activity: &activity, + observations: &crate::effects::Observations::local_session(), + }, ) .await .unwrap(); @@ -3705,8 +3723,10 @@ mod tests { &busy, &mut driver, &sink, - &activity, - &crate::effects::Observations::local_session(), + SessionInstruments { + activity: &activity, + observations: &crate::effects::Observations::local_session(), + }, ) .await; @@ -3780,8 +3800,10 @@ mod tests { &busy, &mut driver, &sink, - &activity, - &crate::effects::Observations::local_session(), + SessionInstruments { + activity: &activity, + observations: &crate::effects::Observations::local_session(), + }, ) .await .unwrap(); @@ -3838,8 +3860,10 @@ mod tests { &AtomicBool::new(false), &mut driver, &sink, - &activity, - &crate::effects::Observations::local_session(), + SessionInstruments { + activity: &activity, + observations: &crate::effects::Observations::local_session(), + }, ) .await; assert!(matches!(result, Err(AcpRuntimeError::ClientClosed))); @@ -3908,8 +3932,10 @@ mod tests { &busy, &mut driver, &sink, - &activity, - &crate::effects::Observations::local_session(), + SessionInstruments { + activity: &activity, + observations: &crate::effects::Observations::local_session(), + }, ) .await; From b650191e3420b0ebe917ce49f29e128f683f0d6f Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 22:35:03 +0100 Subject: [PATCH 6/7] feat(diagnostics): collect opt-in span context in error logs --- .../user/getting-started-and-configuration.md | 26 + docs/user/tui-and-sessions.md | 2 +- src/acp_child.rs | 145 +----- src/effects.rs | 419 ---------------- src/fatal.rs | 347 ++++--------- src/lib.rs | 1 - src/main.rs | 66 +++ src/protocols/a2a.rs | 161 +++---- src/protocols/acp.rs | 437 +++-------------- src/protocols/acp/activity.rs | 50 +- src/protocols/acp/v2.rs | 401 ++-------------- src/runtime.rs | 267 +++------- src/runtime/tests.rs | 260 ++++++---- src/telemetry.rs | 134 +++++- src/telemetry/error_spans.rs | 426 ++++++++++++++++ .../error_spans/task_manager_tests.rs | 207 ++++++++ src/tools/observed.rs | 291 +++++------ src/tools/subagent.rs | 210 +++----- src/tools/subagent/tests.rs | 454 ++---------------- src/tui/mod.rs | 2 + tests/runtime.rs | 4 +- 21 files changed, 1610 insertions(+), 2700 deletions(-) delete mode 100644 src/effects.rs create mode 100644 src/telemetry/error_spans.rs create mode 100644 src/telemetry/error_spans/task_manager_tests.rs diff --git a/docs/user/getting-started-and-configuration.md b/docs/user/getting-started-and-configuration.md index 8289f8fc..fa2f642a 100644 --- a/docs/user/getting-started-and-configuration.md +++ b/docs/user/getting-started-and-configuration.md @@ -157,6 +157,7 @@ provider = "openai-subscription" # or "openrouter" or "speakeasy" model = "gpt-5.4" reasoning_effort = "medium" # low, medium, or high a2a = "127.0.0.1:7331" +capture_error_spans = false # optional local context in fatal error logs otel_endpoint = "http://localhost:4317" otel_protocol = "grpc" # grpc, http/protobuf, or http/json otel_capture_message_content = false @@ -210,6 +211,31 @@ For settings exposed by a command, precedence is: 2. values in `~/.kit/config.toml`; 3. built-in defaults. +Set `capture_error_spans = true` to include a bounded structured span history +alongside existing fatal errors in `~/.kit/errors//`. It defaults to +`false` when omitted. This local collector is independent of OTLP export: +neither setting enables the other. Disabled collection installs no diagnostic +layer and retains no diagnostic span history. Built-in TUI and `acp.kit` children +inherit the resolved setting and must use a compatible Kit executable. + +The optional `span_context` field contains operation-local parent indexes and +allowlisted operation names, launch kinds, tool-error classifications, booleans, +and bounded counts from tracing spans. It excludes external identifiers, +`Debug`/`Display` values, messages, prompts, tool arguments/results, URLs, and +provider payloads, even when OTEL message-content capture is enabled. Attributes +set only through OpenTelemetry APIs are not collected. Histories are limited to +24 fragments, eight descendant levels, six fields per fragment, 32 bytes per +string value, and 12 KiB of serialized context. Existing schema-v2 errors remain +readable; files are not rewritten on read. + +Histories cover instrumented prompt and autonomous operation boundaries, including +already-closed child spans. They are partial operation histories, not proof that +sibling spans caused an error or that replay is safe. Collection is best-effort; +missing observations never prove that no effects occurred. Upstream TaskManager +spawns currently lose tracing ancestry, so tool task bodies and background work +inside those spawns can be absent even when their dispatch span was observed. +Remote child-process spans are not transported into the parent's error log. + The OpenTelemetry endpoint follows the same CLI-over-TOML precedence, then falls back to the standard `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. If none is set, trace export is disabled. The trace protocol precedence is diff --git a/docs/user/tui-and-sessions.md b/docs/user/tui-and-sessions.md index 6f6ecf54..2eb3ab3a 100644 --- a/docs/user/tui-and-sessions.md +++ b/docs/user/tui-and-sessions.md @@ -150,7 +150,7 @@ Kit stores durable JSONL transcripts, locks, and session-associated fatal error The workspace hash is the BLAKE3 digest of the canonical workspace-root path. It keeps identical session IDs in different workspaces in separate storage directories. The optional metadata sidecar stores only the custom display name, is replaced atomically, and does not modify or lock the append-only transcript. Missing or malformed metadata falls back to the generated title without hiding the session. -Fatal error records use their own versioned JSON schema and are not transcript content. Schema v2 supports optional structured transport diagnostics and additive possible-effects observations; schema v1 records remain readable. Missing observations mean unknown activity, not absence of effects. Observations are incomplete and never establish that retrying is safe. Transport diagnostics contain only bounded, allowlisted request/stream stage, retry, attempt, the provider's strictly validated `x-request-id` value, reqwest classification, and typed Hyper, HTTP/2, and I/O fields. Unknown or truncated source chains are identified without storing source text. Kit never stores raw error display/debug text, arbitrary headers, prompts, tool arguments, response bodies, credentials, URLs, or peer-controlled HTTP/2 debug text in these records. Files are written atomically with owner-only permissions on Unix, and Kit retains the newest 50 records per session. Cancellation remains a cancellation outcome, but can create a diagnostic record to retain possible-effects observations. When persistence succeeds, local prompt and ACP terminal errors include the log path; A2A records stay server-local. +Fatal error records use their own versioned JSON schema and are not transcript content. Schema v2 adds optional structured transport diagnostics; schema v1 records remain readable. Transport diagnostics contain only bounded, allowlisted request/stream stage, retry, attempt, the provider's strictly validated `x-request-id` value, reqwest classification, and typed Hyper, HTTP/2, and I/O fields. Unknown or truncated source chains are identified without storing source text. Kit never stores raw error display/debug text, arbitrary headers, prompts, tool arguments, response bodies, credentials, URLs, or peer-controlled HTTP/2 debug text in these records. Files are written atomically with owner-only permissions on Unix, and Kit retains the newest 50 records per session. Cancellation is not a fatal error and does not create a record. When persistence succeeds, local prompt and ACP terminal errors include the log path; A2A records stay server-local. `HOME is unset; cannot locate durable sessions` means Kit cannot determine this directory. Set `HOME` to the intended home directory before starting Kit. diff --git a/src/acp_child.rs b/src/acp_child.rs index 563375a0..cd30eeda 100644 --- a/src/acp_child.rs +++ b/src/acp_child.rs @@ -394,56 +394,23 @@ impl ChildConfig { } } -#[derive(Clone, Debug)] +#[derive(Debug)] pub(crate) enum ChildError { Cancelled, Failed(String), TerminalCancelled, TerminalFailed(String), - Observed { - error: Box, - effects: crate::effects::PossibleEffects, - }, } impl std::fmt::Display for ChildError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Cancelled | Self::TerminalCancelled => f.write_str("nested agent cancelled"), Self::Failed(e) | Self::TerminalFailed(e) => f.write_str(e), - Self::Observed { error, .. } => error.fmt(f), - } - } -} - -impl ChildError { - pub(crate) fn root(&self) -> &Self { - match self { - Self::Observed { error, .. } => error.root(), - error => error, - } - } - - pub(crate) fn possible_effects(&self) -> crate::effects::PossibleEffects { - match self { - Self::Observed { effects, .. } => *effects, - _ => crate::effects::PossibleEffects::default(), - } - } - - fn observed(self, observations: &crate::effects::Observations) -> Self { - self.with_effects(observations.snapshot()) - } - - pub(crate) fn with_effects(self, effects: crate::effects::PossibleEffects) -> Self { - Self::Observed { - error: Box::new(self), - effects, } } } struct Prompt { - observations: crate::effects::Observations, session_id: SessionId, text: String, cancellation: TurnCancellation, @@ -473,7 +440,6 @@ struct Ready { #[derive(Clone, Debug, Default)] pub(crate) struct ChildOutput { - observations: crate::effects::Observations, pub text: String, pub updates: Vec, pub updates_truncated: bool, @@ -481,12 +447,7 @@ pub(crate) struct ChildOutput { } impl ChildOutput { - pub(crate) fn possible_effects(&self) -> crate::effects::PossibleEffects { - self.observations.snapshot() - } - fn record(&mut self, update: SessionUpdate) { - self.observations.record(&update); if let SessionUpdate::AgentMessageChunk(chunk) = &update && let ContentBlock::Text(text) = &chunk.content { @@ -767,30 +728,24 @@ impl ChildSession { text: String, cancellation: TurnCancellation, ) -> Result { - // This owner outlives the prompt task, including channel loss/abort. - let observations = crate::effects::Observations::default(); - let outcome = async { - let _serial = tokio::select! { - serial = self.serial.lock() => serial, - () = cancellation.cancelled() => return Err(ChildError::Cancelled), - }; - let (reply, response) = oneshot::channel(); - let request = Request::Prompt(Prompt { - observations: observations.clone(), - session_id: self.session_id.clone(), - text, - cancellation: cancellation.clone(), - reply, - }); - tokio::select! { - sent = self.tx.send(request) => sent.map_err(|_| ChildError::TerminalFailed("nested agent process is no longer running".into()))?, - () = cancellation.cancelled() => return Err(ChildError::Cancelled), - } - response.await.map_err(|_| { - ChildError::TerminalFailed("nested agent process exited without a response".into()) - })? - }.await; - outcome.map_err(|error| error.observed(&observations)) + let _serial = tokio::select! { + serial = self.serial.lock() => serial, + () = cancellation.cancelled() => return Err(ChildError::Cancelled), + }; + let (reply, response) = oneshot::channel(); + let request = Request::Prompt(Prompt { + session_id: self.session_id.clone(), + text, + cancellation: cancellation.clone(), + reply, + }); + tokio::select! { + sent = self.tx.send(request) => sent.map_err(|_| ChildError::TerminalFailed("nested agent process is no longer running".into()))?, + () = cancellation.cancelled() => return Err(ChildError::Cancelled), + } + response.await.map_err(|_| { + ChildError::TerminalFailed("nested agent process exited without a response".into()) + })? } } @@ -1096,10 +1051,7 @@ async fn run( let fatal = fatal_tx.clone(); tasks.spawn(async move { let session_id = prompt.session_id.clone(); - let output = Arc::new(Mutex::new(ChildOutput { - observations: prompt.observations.clone(), - ..ChildOutput::default() - })); + let output = Arc::new(Mutex::new(ChildOutput::default())); if let Ok(mut routes) = routes.lock() { routes.insert(session_id.clone(), Arc::clone(&output)); } let request = connection.send_request(agentkit_acp::PromptRequest::new( session_id.clone(), vec![ContentBlock::Text(agentkit_acp::TextContent::new(prompt.text))], @@ -1279,63 +1231,6 @@ mod tests { use super::*; - #[test] - fn effect_observations_survive_output_retention_limits() { - let mut output = ChildOutput { - updates: vec![Value::Null; MAX_CAPTURED_UPDATES], - ..ChildOutput::default() - }; - output.record(update(json!({ - "sessionUpdate": "tool_call", "toolCallId": "secret-id", - "title": "secret command", "status": "in_progress" - }))); - assert!(output.updates_truncated); - assert!(output.observations.snapshot().tool_execution_start_reported); - output.record(update(json!({ - "sessionUpdate": "agent_message_chunk", - "content": {"type": "text", "text": "private assistant output"} - }))); - let failure = ChildError::TerminalCancelled.observed(&output.observations); - assert!(failure.possible_effects().assistant_output_observed); - assert!(matches!(failure.root(), ChildError::TerminalCancelled)); - assert!(failure.possible_effects().observation_incomplete); - } - - #[tokio::test] - async fn reply_channel_loss_preserves_positive_observations() { - let (tx, mut rx) = mpsc::channel(1); - let (_closed, closed) = watch::channel(false); - let child = ChildSession { - tx, - session_id: "test".into(), - capabilities: agentkit_acp::AgentCapabilities::default(), - serial: Arc::new(tokio::sync::Mutex::new(())), - closed, - descendant_parent: None, - }; - let actor = tokio::spawn(async move { - let Some(Request::Prompt(prompt)) = rx.recv().await else { - panic!("prompt expected") - }; - prompt.observations.record(&update(json!({ - "sessionUpdate": "tool_call", "toolCallId": "t", - "title": "tool", "status": "completed" - }))); - // Simulate transport/task loss before the response is delivered. - drop(prompt.reply); - }); - let error = child - .prompt("private prompt".into(), TurnCancellation::default()) - .await - .unwrap_err(); - actor.await.unwrap(); - assert!(matches!(error.root(), ChildError::TerminalFailed(_))); - let effects = error.possible_effects(); - assert!(effects.tool_execution_completion_reported); - assert!(!effects.tool_execution_start_reported); - assert!(effects.observation_incomplete); - } - fn update(value: Value) -> SessionUpdate { serde_json::from_value(value).unwrap() } diff --git a/src/effects.rs b/src/effects.rs deleted file mode 100644 index f6088b1f..00000000 --- a/src/effects.rs +++ /dev/null @@ -1,419 +0,0 @@ -//! Bounded positive observations, never a replay-safety decision. -//! ACP lifecycle statuses are reports from the harness, not execution receipts. - -use std::{ - collections::HashSet, - sync::{Arc, Mutex}, -}; - -use agentkit_core::{Delta, Part, PartId, PartKind}; -use agentkit_loop::{AgentEvent, LoopObserver, ObservedEvent}; - -use agentkit_acp::{SessionUpdate, ToolCallStatus}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub(crate) enum ObservationSource { - #[default] - Unknown, - AcpNotifications, - /// Cumulative observations during this live root owner's lifetime only. - LocalSession, -} - -/// False means only "not observed". Completion does not imply success, -/// rollback, a committed effect, or permission to repeat an operation. -#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub(crate) struct PossibleEffects { - pub source: ObservationSource, - pub assistant_output_observed: bool, - pub tool_emission_observed: bool, - pub tool_execution_start_reported: bool, - pub tool_execution_completion_reported: bool, - #[serde(deserialize_with = "incomplete_observation")] - pub observation_incomplete: bool, -} - -fn incomplete_observation<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result { - if bool::deserialize(deserializer)? { - Ok(true) - } else { - Err(serde::de::Error::custom( - "failure observation must remain incomplete", - )) - } -} - -impl Default for PossibleEffects { - fn default() -> Self { - Self { - source: ObservationSource::Unknown, - assistant_output_observed: false, - tool_emission_observed: false, - tool_execution_start_reported: false, - tool_execution_completion_reported: false, - // No failure path proves complete observation of external activity. - observation_incomplete: true, - } - } -} - -const MAX_TRACKED_PARTS: usize = 128; -const MAX_PART_ID_BYTES: usize = 256; - -#[derive(Debug, Default)] -struct ObservationState { - effects: PossibleEffects, - // Transient classification only; never serialized into diagnostics. - assistant_parts: HashSet, -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct Observations(Arc>); - -impl Observations { - pub(crate) fn local_session() -> Self { - Self(Arc::new(Mutex::new(ObservationState { - effects: PossibleEffects { - source: ObservationSource::LocalSession, - ..Default::default() - }, - ..Default::default() - }))) - } - - pub(crate) fn invocation_started(&self) { - if let Ok(mut state) = self.0.lock() { - state.effects.tool_execution_start_reported = true; - } - } - - pub(crate) fn invocation_completed(&self) { - if let Ok(mut state) = self.0.lock() { - state.effects.tool_execution_completion_reported = true; - } - } - - fn observe_local(&self, event: &AgentEvent) { - let Ok(mut state) = self.0.lock() else { return }; - match event { - AgentEvent::ToolCallRequested(_) => state.effects.tool_emission_observed = true, - AgentEvent::ContentDelta(delta) => match delta { - Delta::BeginPart { part_id, kind } => { - state.assistant_parts.remove(part_id); - if *kind == PartKind::ToolCall { - state.effects.tool_emission_observed = true; - } - if matches!( - kind, - PartKind::Text | PartKind::Media | PartKind::File | PartKind::Structured - ) && part_id.0.len() <= MAX_PART_ID_BYTES - && state.assistant_parts.len() < MAX_TRACKED_PARTS - { - state.assistant_parts.insert(part_id.clone()); - } - } - Delta::AppendText { part_id, chunk } => { - if !chunk.is_empty() && state.assistant_parts.contains(part_id) { - state.effects.assistant_output_observed = true; - } - } - Delta::AppendBytes { part_id, chunk } => { - if !chunk.is_empty() && state.assistant_parts.contains(part_id) { - state.effects.assistant_output_observed = true; - } - } - Delta::ReplaceStructured { part_id, .. } => { - if state.assistant_parts.contains(part_id) { - state.effects.assistant_output_observed = true; - } - } - Delta::CommitPart { part } => { - match part { - Part::Text(text) if !text.text.is_empty() => { - state.effects.assistant_output_observed = true - } - Part::Media(_) | Part::File(_) | Part::Structured(_) => { - state.effects.assistant_output_observed = true - } - Part::ToolCall(_) => state.effects.tool_emission_observed = true, - _ => {} - } - // CommitPart carries no PartId. Forget classifications rather - // than risk applying a stale kind to a later reused id. - state.assistant_parts.clear(); - } - Delta::SetMetadata { .. } => {} - }, - AgentEvent::TurnStarted { .. } - | AgentEvent::TurnFinished(_) - | AgentEvent::ResponseAttemptSuperseded => { - // Presentation/turn boundaries cannot erase effects, particularly - // when a detached invocation spans more than one prompt. - state.assistant_parts.clear(); - } - // ToolResultReceived includes synthetic permission/cancellation - // results. Only actual invocation boundaries report execution. - _ => {} - } - } - - pub(crate) fn snapshot(&self) -> PossibleEffects { - self.0.lock().map(|value| value.effects).unwrap_or_default() - } - - pub(crate) fn record(&self, update: &SessionUpdate) { - let Ok(mut state) = self.0.lock() else { - return; - }; - let effects = &mut state.effects; - effects.source = ObservationSource::AcpNotifications; - let status = match update { - SessionUpdate::AgentMessageChunk(_) => { - effects.assistant_output_observed = true; - None - } - SessionUpdate::ToolCall(call) => { - effects.tool_emission_observed = true; - Some(call.status) - } - SessionUpdate::ToolCallUpdate(update) => { - effects.tool_emission_observed = true; - update.fields.status - } - _ => None, - }; - match status { - Some(ToolCallStatus::InProgress) => effects.tool_execution_start_reported = true, - Some(ToolCallStatus::Completed) => { - // Do not manufacture a start observation from completion. - // Failed can mean pre-execution denial, not completed execution. - effects.tool_execution_completion_reported = true; - } - _ => {} - } - } -} - -impl LoopObserver for Observations { - fn handle_event(&self, event: ObservedEvent) { - self.observe_local(&event.event); - } -} - -#[cfg(test)] -pub(crate) fn isolated_test(name: &str) -> bool { - if std::env::var("KIT_EFFECTS_TEST_CHILD").as_deref() == Ok(name) { - return false; - } - let home = tempfile::tempdir().unwrap(); - let output = std::process::Command::new(std::env::current_exe().unwrap()) - .args(["--exact", name, "--nocapture"]) - .env("KIT_EFFECTS_TEST_CHILD", name) - .env("HOME", home.path()) - .env_remove(crate::events::EVENTS_ENV) - .output() - .unwrap(); - assert!( - output.status.success(), - "isolated effects test failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - true -} - -#[cfg(test)] -pub(crate) fn test_record(session_id: &str) -> serde_json::Value { - let directory = std::path::PathBuf::from(std::env::var_os("HOME").unwrap()) - .join(".kit/errors") - .join(session_id); - let mut files: Vec<_> = std::fs::read_dir(directory) - .unwrap() - .map(|file| file.unwrap().path()) - .filter(|path| path.extension().is_some_and(|ext| ext == "json")) - .collect(); - files.sort(); - serde_json::from_slice(&std::fs::read(files.last().unwrap()).unwrap()).unwrap() -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - fn update(value: serde_json::Value) -> SessionUpdate { - serde_json::from_value(value).unwrap() - } - - #[test] - fn observations_are_monotonic_and_do_not_infer_execution() { - let observations = Observations::default(); - assert_eq!(observations.snapshot(), PossibleEffects::default()); - observations.record(&update(json!({ - "sessionUpdate": "agent_message_chunk", - "content": {"type": "text", "text": "secret output"} - }))); - assert!(observations.snapshot().assistant_output_observed); - observations.record(&update(json!({ - "sessionUpdate": "tool_call", "toolCallId": "private-id", - "title": "private command", "status": "pending", - "rawInput": {"secret": "private arguments"} - }))); - let emitted = observations.snapshot(); - assert!(emitted.tool_emission_observed); - assert!(!emitted.tool_execution_start_reported); - observations.record(&update(json!({ - "sessionUpdate": "tool_call_update", "toolCallId": "private-id", - "status": "completed", "rawOutput": "private result" - }))); - let completed = observations.snapshot(); - assert!(completed.tool_execution_completion_reported); - assert!(!completed.tool_execution_start_reported); - observations.record(&update(json!({ - "sessionUpdate": "tool_call_update", "toolCallId": "other", - "status": "in_progress" - }))); - let effects = observations.snapshot(); - assert!(effects.tool_execution_start_reported); - assert!(effects.tool_execution_completion_reported); - assert!(effects.observation_incomplete); - let envelope = serde_json::to_string(&effects).unwrap(); - assert!(!envelope.contains("secret")); - assert!(!envelope.contains("private")); - assert!(envelope.len() < 1024); - } - - #[test] - fn failed_status_does_not_prove_execution_completed() { - let observations = Observations::default(); - observations.record(&update(json!({ - "sessionUpdate": "tool_call_update", "toolCallId": "denied", - "status": "failed" - }))); - let effects = observations.snapshot(); - assert!(effects.tool_emission_observed); - assert!(!effects.tool_execution_start_reported); - assert!(!effects.tool_execution_completion_reported); - assert!(effects.observation_incomplete); - } - - #[test] - fn malformed_or_payload_bearing_metadata_is_rejected() { - let mut complete = serde_json::to_value(PossibleEffects::default()).unwrap(); - complete["observation_incomplete"] = json!(false); - assert!(serde_json::from_value::(complete).is_err()); - let mut value = serde_json::to_value(PossibleEffects::default()).unwrap(); - value["tool_arguments"] = json!("secret"); - assert!(serde_json::from_value::(value).is_err()); - let mut value = serde_json::to_value(PossibleEffects::default()).unwrap(); - value["assistant_output_observed"] = json!("false"); - assert!(serde_json::from_value::(value).is_err()); - } - #[test] - fn local_content_classification_is_bounded_and_never_invents_output() { - let observations = Observations::local_session(); - let emit = |delta| observations.observe_local(&AgentEvent::ContentDelta(delta)); - let id = PartId::new("private-part"); - emit(Delta::BeginPart { - part_id: id.clone(), - kind: PartKind::Reasoning, - }); - emit(Delta::AppendText { - part_id: id.clone(), - chunk: "private reasoning".into(), - }); - emit(Delta::BeginPart { - part_id: id.clone(), - kind: PartKind::ToolCall, - }); - emit(Delta::AppendText { - part_id: id.clone(), - chunk: "private arguments".into(), - }); - assert!(!observations.snapshot().assistant_output_observed); - assert!(observations.snapshot().tool_emission_observed); - assert!(!observations.snapshot().tool_execution_start_reported); - emit(Delta::BeginPart { - part_id: id.clone(), - kind: PartKind::Text, - }); - assert!(!observations.snapshot().assistant_output_observed); - emit(Delta::AppendText { - part_id: id, - chunk: "private answer".into(), - }); - assert!(observations.snapshot().assistant_output_observed); - observations.observe_local(&AgentEvent::ResponseAttemptSuperseded); - observations.observe_local(&AgentEvent::TurnStarted { - session_id: "session".into(), - turn_id: "next".into(), - }); - assert!(observations.snapshot().assistant_output_observed); - let encoded = serde_json::to_string(&observations.snapshot()).unwrap(); - assert!(!encoded.contains("private")); - assert_eq!( - observations.snapshot().source, - ObservationSource::LocalSession - ); - assert!(observations.snapshot().observation_incomplete); - - let bounded = Observations::local_session(); - for index in 0..MAX_TRACKED_PARTS + 3 { - bounded.observe_local(&AgentEvent::ContentDelta(Delta::BeginPart { - part_id: format!("part-{index}").into(), - kind: PartKind::Text, - })); - } - assert_eq!( - bounded.0.lock().unwrap().assistant_parts.len(), - MAX_TRACKED_PARTS - ); - bounded.observe_local(&AgentEvent::ContentDelta(Delta::AppendText { - part_id: format!("part-{}", MAX_TRACKED_PARTS + 2).into(), - chunk: "unclassified".into(), - })); - let huge = PartId::new("x".repeat(MAX_PART_ID_BYTES + 1)); - bounded.observe_local(&AgentEvent::ContentDelta(Delta::BeginPart { - part_id: huge.clone(), - kind: PartKind::Text, - })); - bounded.observe_local(&AgentEvent::ContentDelta(Delta::AppendText { - part_id: huge, - chunk: "unclassified".into(), - })); - assert!(!bounded.snapshot().assistant_output_observed); - bounded.observe_local(&AgentEvent::ContentDelta(Delta::CommitPart { - part: Part::text("commit-only output"), - })); - assert!(bounded.snapshot().assistant_output_observed); - assert!(bounded.0.lock().unwrap().assistant_parts.is_empty()); - } - - #[test] - fn synthesized_results_are_not_invocation_receipts_and_sessions_are_isolated() { - let first = Observations::local_session(); - let second = Observations::local_session(); - first.observe_local(&AgentEvent::ToolResultReceived( - agentkit_core::ToolResultPart::error( - "denied", - agentkit_core::ToolOutput::text("denied"), - ), - )); - assert!(!first.snapshot().tool_execution_start_reported); - assert!(!first.snapshot().tool_execution_completion_reported); - let background = first.clone(); - background.invocation_started(); - first.observe_local(&AgentEvent::TurnStarted { - session_id: "session".into(), - turn_id: "next".into(), - }); - background.invocation_completed(); - assert!(first.snapshot().tool_execution_start_reported); - assert!(first.snapshot().tool_execution_completion_reported); - assert!(!second.snapshot().tool_execution_start_reported); - } -} diff --git a/src/fatal.rs b/src/fatal.rs index 1b5e74d3..996e24db 100644 --- a/src/fatal.rs +++ b/src/fatal.rs @@ -25,7 +25,6 @@ pub(crate) enum Surface { A2a, Acp, Prompt, - Subagent, } impl Surface { @@ -34,7 +33,6 @@ impl Surface { Self::A2a => "a2a", Self::Acp => "acp", Self::Prompt => "prompt", - Self::Subagent => "subagent", } } } @@ -52,8 +50,22 @@ struct FatalRecord { message: String, #[serde(default, skip_serializing_if = "Option::is_none")] diagnostics: Option, - #[serde(default)] - possible_effects: crate::effects::PossibleEffects, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_span_context" + )] + span_context: Option, +} + +fn deserialize_span_context<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let snapshot = Option::::deserialize(deserializer)?; + if snapshot.as_ref().is_some_and(|snapshot| !snapshot.valid()) { + return Err(serde::de::Error::custom("invalid span context")); + } + Ok(snapshot) } #[derive(Clone, Copy, Debug, Deserialize, Serialize)] @@ -230,51 +242,37 @@ fn split_diagnostics(message: &str) -> (&str, Option) { (plain, Some(diagnostics)) } -/// Explicit observation owner, including cancellation without reclassifying it. -pub(crate) fn record_loop_error_with_effects( +pub(crate) fn record_loop_error( session_id: &str, surface: Surface, error: &LoopError, - effects: crate::effects::PossibleEffects, ) -> Result, String> { - let (kind, code, message, diagnostics) = if matches!(error, LoopError::Cancelled) { - ( - "cancelled", - "cancelled", - "execution cancelled; effects and completion are unconfirmed".into(), - None, - ) - } else if let Some(classified) = classify(error) { - classified - } else { + let Some((kind, code, message, diagnostics)) = classify(error) else { return Ok(None); }; - write_default_with_effects( + write_default( session_id, surface, kind, code, &message, diagnostics.as_ref(), - effects, ) .map(Some) } -pub(crate) fn record_runtime_error_with_effects( +pub(crate) fn record_runtime_error( session_id: &str, surface: Surface, code: &str, - effects: crate::effects::PossibleEffects, ) -> Result { - write_default_with_effects( + write_default( session_id, surface, "runtime", canonical_code(code), "runtime failed before the session could continue", None, - effects, ) } @@ -412,29 +410,25 @@ fn canonical_code(code: &str) -> &str { } } -fn write_default_with_effects( +fn write_default( session_id: &str, surface: Surface, kind: &str, code: &str, message: &str, diagnostics: Option<&TransportDiagnostics>, - effects: crate::effects::PossibleEffects, ) -> Result { let home = std::env::var_os("HOME") .filter(|home| !home.is_empty()) .ok_or_else(|| "HOME is unset; cannot store fatal error log".to_owned())?; - write_in_with_effects( + write_in_with_diagnostics( &PathBuf::from(home).join(".kit/errors"), session_id, surface, - FatalDetails { - kind, - code, - message, - }, + kind, + code, + message, diagnostics, - effects, ) } @@ -450,7 +444,6 @@ fn write_in( write_in_with_diagnostics(base, session_id, surface, kind, code, message, None) } -#[cfg(test)] fn write_in_with_diagnostics( base: &Path, session_id: &str, @@ -459,49 +452,6 @@ fn write_in_with_diagnostics( code: &str, message: &str, diagnostics: Option<&TransportDiagnostics>, -) -> Result { - write_in_with_effects( - base, - session_id, - surface, - FatalDetails { - kind, - code, - message, - }, - diagnostics, - crate::effects::PossibleEffects::default(), - ) -} - -pub(crate) fn record_child_failure( - session_id: &str, - effects: crate::effects::PossibleEffects, -) -> Result { - write_default_with_effects( - session_id, - Surface::Subagent, - "runtime", - "subagent_failed", - "nested agent failed; effects and completion are unconfirmed", - None, - effects, - ) -} - -struct FatalDetails<'a> { - kind: &'a str, - code: &'a str, - message: &'a str, -} - -fn write_in_with_effects( - base: &Path, - session_id: &str, - surface: Surface, - details: FatalDetails<'_>, - diagnostics: Option<&TransportDiagnostics>, - possible_effects: crate::effects::PossibleEffects, ) -> Result { crate::session::validate_id(session_id)?; let occurred_at_ms = SystemTime::now() @@ -513,21 +463,26 @@ fn write_in_with_effects( std::process::id(), NEXT_EVENT.fetch_add(1, Ordering::Relaxed) ); - let record = FatalRecord { + let mut record = FatalRecord { schema_version: SCHEMA_VERSION, event_id: event_id.clone(), occurred_at_ms, kit_version: env!("CARGO_PKG_VERSION").into(), session_id: session_id.into(), surface: surface.as_str().into(), - kind: details.kind.into(), - code: canonical_code(details.code).into(), - message: bounded(details.message), + kind: kind.into(), + code: canonical_code(code).into(), + message: bounded(message), diagnostics: diagnostics.filter(|value| value.valid()).cloned(), - possible_effects, + span_context: crate::telemetry::error_spans::snapshot(&tracing::Span::current()), }; let mut bytes = serde_json::to_vec_pretty(&record) .map_err(|error| format!("could not encode fatal error log: {error}"))?; + if bytes.len() >= MAX_RECORD_BYTES && record.span_context.take().is_some() { + // Optional diagnostics must not displace an otherwise valid ordinary error. + bytes = serde_json::to_vec_pretty(&record) + .map_err(|error| format!("could not encode fatal error log: {error}"))?; + } bytes.push(b'\n'); if bytes.len() > MAX_RECORD_BYTES { return Err("fatal error log exceeds size limit".into()); @@ -610,8 +565,8 @@ mod tests { use super::{ DIAGNOSTIC_MARKER, FatalRecord, H2Reason, IoClassification, MAX_DIAGNOSTIC_BYTES, MAX_RECORDS_PER_SESSION, ReqwestDiagnostics, Surface, TransportDiagnostics, - TransportSource, TransportStage, bounded, classify, event_order, render_loop_error, - split_diagnostics, write_in, write_in_with_diagnostics, + TransportSource, TransportStage, bounded, classify, event_order, record_loop_error, + render_loop_error, split_diagnostics, write_in, write_in_with_diagnostics, }; fn append_diagnostics(message: String, diagnostics: &TransportDiagnostics) -> String { @@ -688,51 +643,11 @@ mod tests { } #[test] - fn writes_effects_without_sensitive_content_and_reads_legacy_unknown() { - let root = tempfile::tempdir().unwrap(); - let effects = crate::effects::PossibleEffects { - source: crate::effects::ObservationSource::AcpNotifications, - assistant_output_observed: true, - ..crate::effects::PossibleEffects::default() - }; - let path = super::write_in_with_effects( - root.path(), - "session-effects", - Surface::Subagent, - super::FatalDetails { - kind: "runtime", - code: "subagent_failed", - message: "nested agent failed", - }, - None, - effects, - ) - .unwrap(); - let mut value: serde_json::Value = - serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - let current: FatalRecord = serde_json::from_value(value.clone()).unwrap(); - assert_eq!(current.possible_effects, effects); - assert_eq!(current.schema_version, 2); - for marker in [1, 2, 3, 4] { - value["schema_version"] = json!(marker); - value.as_object_mut().unwrap().remove("possible_effects"); - let legacy: FatalRecord = serde_json::from_value(value.clone()).unwrap(); - assert_eq!( - legacy.possible_effects, - crate::effects::PossibleEffects::default() - ); - // Presence, not the numeric marker, determines whether observations exist. - value["possible_effects"] = serde_json::to_value(effects).unwrap(); - let extended: FatalRecord = serde_json::from_value(value.clone()).unwrap(); - assert_eq!(extended.possible_effects, effects); - } - } - - #[test] - fn frozen_v2_reader_retains_known_fields_from_extended_records() { - // Pre-effects top-level reader; the transport diagnostics shape is unchanged. - #[derive(Debug, serde::Deserialize, serde::Serialize)] - struct FatalRecordV2 { + fn schema_two_readers_preserve_optional_span_context() { + use tracing_subscriber::prelude::*; + // Frozen shipped schema-v2 shape: unknown top-level fields are ignored. + #[derive(serde::Deserialize, serde::Serialize)] + struct ShippedV2 { schema_version: u64, event_id: String, occurred_at_ms: u64, @@ -743,46 +658,66 @@ mod tests { code: String, message: String, #[serde(default, skip_serializing_if = "Option::is_none")] - diagnostics: Option, + diagnostics: Option, } - let root = tempfile::tempdir().unwrap(); - let diagnostics = sample_diagnostics(); - for source in [ - crate::effects::ObservationSource::Unknown, - crate::effects::ObservationSource::AcpNotifications, - crate::effects::ObservationSource::LocalSession, - ] { - let effects = crate::effects::PossibleEffects { - source, - tool_execution_completion_reported: true, - ..Default::default() - }; - let path = super::write_in_with_effects( - root.path(), - "session-extended", - Surface::Subagent, - super::FatalDetails { - kind: "runtime", - code: "subagent_failed", - message: "nested agent failed", - }, - Some(&diagnostics), - effects, - ) - .unwrap(); - let bytes = fs::read(path).unwrap(); - let legacy: FatalRecordV2 = serde_json::from_slice(&bytes).unwrap(); - let current: FatalRecord = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(legacy.schema_version, 2); - assert_eq!(current.possible_effects, effects); - let mut known_fields: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - known_fields - .as_object_mut() - .unwrap() - .remove("possible_effects"); - assert_eq!(serde_json::to_value(legacy).unwrap(), known_fields); - } + tracing::subscriber::with_default( + tracing_subscriber::registry().with(crate::telemetry::error_spans::ErrorSpanLayer), + || { + let operation = crate::telemetry::error_spans::operation("prompt"); + operation.in_scope(|| { + { let _child = tracing::info_span!(target: "agentkit_loop", "agent.execute_tool", launch_kind = "plain"); } + let path = write_in_with_diagnostics(root.path(), "session-context", Surface::Prompt, "provider", "stream_transport", "openai-subscription stream transport failed", Some(&sample_diagnostics())).unwrap(); + let bytes = fs::read(&path).unwrap(); + let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(value["schema_version"], 2); + assert_eq!(value["span_context"]["fragments"][1]["fields"]["launch_kind"], "plain"); + let current: FatalRecord = serde_json::from_slice(&bytes).unwrap(); + let legacy: ShippedV2 = serde_json::from_slice(&bytes).unwrap(); + let mut known = serde_json::to_value(¤t).unwrap(); + known.as_object_mut().unwrap().remove("span_context"); + assert_eq!(serde_json::to_value(&legacy).unwrap(), known); + assert_eq!(current.message, "openai-subscription stream transport failed"); + let supplied = value["span_context"].clone(); + for marker in [1, 2, 3, 4] { + value["schema_version"] = json!(marker); + let parsed: FatalRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(parsed.span_context).unwrap(), supplied); + } + value.as_object_mut().unwrap().remove("span_context"); + assert!(serde_json::from_value::(value.clone()).unwrap().span_context.is_none()); + value["span_context"] = supplied; + value["span_context"]["fragments"][1]["fields"]["launch_kind"] = json!("SECRET"); + assert!(serde_json::from_value::(value).is_err()); + assert_eq!(fs::read(path).unwrap(), bytes); + assert!(bytes.len() < super::MAX_RECORD_BYTES); + }); + }, + ); + } + + #[test] + fn ordinary_writer_omits_disabled_context() { + tracing::subscriber::with_default(tracing_subscriber::registry(), || { + let root = tempfile::tempdir().unwrap(); + let operation = crate::telemetry::error_spans::operation("prompt"); + let path = operation + .in_scope(|| { + write_in( + root.path(), + "session-disabled", + Surface::Prompt, + "runtime", + "runtime_error", + "ordinary error", + ) + }) + .unwrap(); + let value: serde_json::Value = + serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert!(value.get("span_context").is_none()); + assert_eq!(value["message"], "ordinary error"); + }); } #[test] @@ -949,8 +884,9 @@ mod tests { } #[test] - fn cancellation_is_distinct_from_provider_failure_classification() { - assert!(classify(&LoopError::Cancelled).is_none()); + fn cancellation_is_not_recorded() { + let result = record_loop_error("session-1", Surface::Acp, &LoopError::Cancelled).unwrap(); + assert!(result.is_none()); } #[test] @@ -1027,79 +963,4 @@ mod tests { 0o700 ); } - #[test] - fn local_cancellation_has_conservative_post_cleanup_metadata() { - if crate::effects::isolated_test( - "fatal::tests::local_cancellation_has_conservative_post_cleanup_metadata", - ) { - return; - } - let observations = crate::effects::Observations::local_session(); - observations.invocation_started(); - let cleanup = observations.clone(); - cleanup.invocation_completed(); - let path = super::record_loop_error_with_effects( - "root-cancelled", - Surface::Acp, - &LoopError::Cancelled, - observations.snapshot(), - ) - .unwrap() - .unwrap(); - let value: serde_json::Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - assert_eq!(value["schema_version"], 2); - assert_eq!(value["kind"], "cancelled"); - assert_eq!(value["code"], "cancelled"); - assert_eq!(value["possible_effects"]["source"], "local_session"); - assert_eq!( - value["possible_effects"]["tool_execution_completion_reported"], - true - ); - assert_eq!(value["possible_effects"]["observation_incomplete"], true); - } - - #[test] - fn supplied_effects_remain_strict() { - let root = tempfile::tempdir().unwrap(); - let path = write_in( - root.path(), - "session-schema", - Surface::Prompt, - "runtime", - "failed", - "failed", - ) - .unwrap(); - let mut value: serde_json::Value = - serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - for source in ["unknown", "acp_notifications", "local_session"] { - value["possible_effects"]["source"] = json!(source); - let record: FatalRecord = serde_json::from_value(value.clone()).unwrap(); - assert_eq!( - serde_json::to_value(record.possible_effects).unwrap()["source"], - source - ); - } - for (field, invalid) in [ - ("source", json!("unknown-future-source")), - ("assistant_output_observed", json!("true")), - ("observation_incomplete", json!(false)), - ("private_payload", json!("secret")), - ] { - let mut malformed = value.clone(); - malformed["possible_effects"][field] = invalid; - assert!( - serde_json::from_value::(malformed).is_err(), - "{field}" - ); - } - for invalid in [ - serde_json::Value::Null, - json!({"source": "local_session"}), - json!([]), - ] { - value["possible_effects"] = invalid; - assert!(serde_json::from_value::(value.clone()).is_err()); - } - } } diff --git a/src/lib.rs b/src/lib.rs index df2dd21a..b800e72e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,6 @@ mod compose_output; pub mod config_files; mod credentials; pub mod docs; -mod effects; pub mod events; mod fatal; mod file_search; diff --git a/src/main.rs b/src/main.rs index 4d06877b..7c0a4666 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,6 +52,9 @@ fn resolve_openrouter_api_key( #[derive(Args)] struct TelemetryArgs { + /// Resolved local diagnostic setting inherited by built-in Kit children. + #[arg(long, hide = true, global = true, value_name = "BOOL", action = clap::ArgAction::Set)] + internal_capture_error_spans: Option, /// OTLP collector endpoint for OpenTelemetry trace export. #[arg(long, global = true)] otel_endpoint: Option, @@ -232,6 +235,7 @@ struct Config { provider: Option, reasoning_effort: Option, a2a: Option, + capture_error_spans: Option, otel_endpoint: Option, otel_protocol: Option, otel_capture_message_content: Option, @@ -409,6 +413,13 @@ impl Config { max_messages, max_bytes, ) + .map(|mut settings| { + settings.capture_error_spans = args + .internal_capture_error_spans + .or(self.capture_error_spans) + .unwrap_or(false); + settings + }) .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error)) } @@ -1679,6 +1690,61 @@ credential_store = "keychain" assert!(toml::from_str::("otel_protocol = 'http'").is_err()); } + #[test] + fn inherited_error_capture_overrides_config_without_rewriting_it() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("config.toml"); + for inherited in [false, true] { + let text = format!( + "# User-owned settings\ncapture_error_spans = {}\n", + !inherited + ); + fs::write(&path, &text).unwrap(); + let config = Config::load(&path).unwrap(); + let cli = Cli::try_parse_from([ + "kit", + "prompt", + "--internal-capture-error-spans", + &inherited.to_string(), + "hello", + ]) + .unwrap(); + let settings = config + .telemetry_settings(&cli.telemetry, None, None, None, None) + .unwrap(); + assert_eq!(settings.capture_error_spans, inherited); + assert_eq!(fs::read_to_string(&path).unwrap(), text); + } + } + + #[test] + fn error_span_capture_defaults_off_and_is_independent_of_export() { + let cli = Cli::try_parse_from(["kit", "prompt", "hello"]).unwrap(); + for (text, expected) in [ + ("", false), + ("capture_error_spans = false", false), + ("capture_error_spans = true", true), + ] { + let config: Config = toml::from_str(text).unwrap(); + for endpoint in [None, Some("http://localhost:4317".to_owned())] { + let settings = config + .telemetry_settings( + &cli.telemetry, + endpoint.clone(), + Some("true".into()), + None, + None, + ) + .unwrap(); + assert_eq!(settings.capture_error_spans, expected); + assert_eq!(settings.endpoint, endpoint); + assert!(settings.capture_message_content); + } + } + assert!(toml::from_str::("capture_error_spans = 'true'").is_err()); + assert!(toml::from_str::("capture_error_spans = 1").is_err()); + } + #[test] fn telemetry_environment_is_strict_and_settings_are_bounded() { let config = Config::default(); diff --git a/src/protocols/a2a.rs b/src/protocols/a2a.rs index ca529362..2c7e9085 100644 --- a/src/protocols/a2a.rs +++ b/src/protocols/a2a.rs @@ -13,88 +13,82 @@ use a2a_protocol_types::{ }; use sha2::{Digest as _, Sha256}; +use tracing::Instrument as _; use crate::runtime::Runtime; struct KitAgent(Arc); -fn record_failure( - session_id: &str, - error: &agentkit_loop::LoopError, - observations: &crate::effects::Observations, -) -> String { - let rendered = crate::fatal::render_loop_error(error); - match crate::fatal::record_loop_error_with_effects( - session_id, - crate::fatal::Surface::A2a, - error, - observations.snapshot(), - ) { - Ok(Some(path)) => eprintln!( - "stored fatal error log for {session_id}: {}", - path.display() - ), - Ok(None) => {} - Err(log_error) => { - eprintln!("could not store fatal error log for {session_id}: {log_error}") - } - } - rendered -} - impl AgentExecutor for KitAgent { fn execute<'a>( &'a self, context: &'a RequestContext, queue: &'a dyn EventQueueWriter, ) -> Pin> + Send + 'a>> { - Box::pin(async move { - let emit = EventEmitter::new(context, queue); - emit.status(TaskState::Working).await?; - let prompt = context - .message - .parts - .iter() - .filter_map(Part::text_content) - .collect::>() - .join("\n"); - if prompt.trim().is_empty() { - emit.artifact( - "error", - vec![Part::text("A2A request must contain a text part")], - None, - Some(true), - ) - .await?; - emit.status(TaskState::Failed).await?; - return Ok(()); - } - let observations = crate::effects::Observations::local_session(); - match self - .0 - .run_cancelled_observed( - prompt, - 0, - Some(context.cancellation_token.clone()), - observations.clone(), - ) - .await - { - Ok(output) => { - emit.artifact("result", vec![Part::text(output)], None, Some(true)) - .await?; - emit.status(TaskState::Completed).await?; - } - Err(error) => { - let session_id = a2a_session_id(context); - let rendered = record_failure(&session_id, &error, &observations); - emit.artifact("error", vec![Part::text(rendered)], None, Some(true)) - .await?; + Box::pin( + async move { + let emit = EventEmitter::new(context, queue); + emit.status(TaskState::Working).await?; + let prompt = context + .message + .parts + .iter() + .filter_map(Part::text_content) + .collect::>() + .join("\n"); + if prompt.trim().is_empty() { + emit.artifact( + "error", + vec![Part::text("A2A request must contain a text part")], + None, + Some(true), + ) + .await?; emit.status(TaskState::Failed).await?; + return Ok(()); } + match self + .0 + .run_cancelled(prompt, 0, Some(context.cancellation_token.clone())) + .await + { + Ok(output) => { + emit.artifact("result", vec![Part::text(output)], None, Some(true)) + .await?; + emit.status(TaskState::Completed).await?; + } + Err(error) => { + let session_id = a2a_session_id(context); + let rendered = crate::fatal::render_loop_error(&error); + let rendered = match crate::fatal::record_loop_error( + &session_id, + crate::fatal::Surface::A2a, + &error, + ) { + Ok(Some(path)) => { + eprintln!( + "stored fatal error log for {session_id}: {}", + path.display() + ); + rendered + } + Ok(None) => rendered, + Err(log_error) => { + eprintln!( + "could not store fatal error log for {session_id}: {log_error}" + ); + rendered + } + }; + emit.artifact("error", vec![Part::text(rendered)], None, Some(true)) + .await?; + emit.status(TaskState::Failed).await?; + } + } + Ok(()) } - Ok(()) - }) + .instrument(crate::telemetry::error_spans::operation("a2a")), + ) } } @@ -173,34 +167,3 @@ pub(crate) fn dispatcher( ); Ok(JsonRpcDispatcher::new(handler)) } - -#[cfg(test)] -mod effects_tests { - #[tokio::test] - async fn a2a_cancelled_execution_retains_the_captured_owner() { - if crate::effects::isolated_test( - "protocols::a2a::effects_tests::a2a_cancelled_execution_retains_the_captured_owner", - ) { - return; - } - let observations = crate::effects::Observations::local_session(); - let executing = observations.clone(); - let result = async move { - executing.invocation_started(); - tokio::task::yield_now().await; - executing.invocation_completed(); - Err::<(), _>(agentkit_loop::LoopError::Cancelled) - } - .await; - super::record_failure("a2a-effects", &result.unwrap_err(), &observations); - let record = crate::effects::test_record("a2a-effects"); - assert_eq!(record["surface"], "a2a"); - assert_eq!(record["kind"], "cancelled"); - assert_eq!(record["possible_effects"]["source"], "local_session"); - assert_eq!( - record["possible_effects"]["tool_execution_completion_reported"], - true - ); - assert_eq!(record["possible_effects"]["observation_incomplete"], true); - } -} diff --git a/src/protocols/acp.rs b/src/protocols/acp.rs index d295a5be..bf01f4a8 100644 --- a/src/protocols/acp.rs +++ b/src/protocols/acp.rs @@ -45,6 +45,7 @@ use tokio::{ task::{AbortHandle, JoinSet}, time::timeout, }; +use tracing::Instrument as _; mod activity; mod skill_catalog; @@ -1325,10 +1326,20 @@ impl Server { cancellation: handle.cancellation_handle(), response_attempt_replacement: true, }; - let driver = self + let driver = match self .runtime .start_acp_driver_with_initial(context, &mut claim, forked) - .await?; + .await + { + Ok(driver) => driver, + Err(error) => { + return Err(record_acp_runtime_failure( + &session_id, + "session_start", + error, + )); + } + }; let current = driver .adapter .selection() @@ -1640,7 +1651,9 @@ async fn session_actor(actor: SessionActor) { &tasks, &background_jobs, structured_completion, - ), |reason| Some(reason.clone())).await; + ), |reason| Some(reason.clone())) + .instrument(crate::telemetry::error_spans::operation("acp")) + .await; let response = result.and_then(|reason| { agentkit_acp::finish_reason_to_stop_reason(&reason).map(PromptResponse::new) }); @@ -1695,7 +1708,6 @@ async fn session_actor(actor: SessionActor) { &integration, &mut driver, &activity, - &background_jobs.observations, ).await, Err(error) => Err(error), }; @@ -1715,7 +1727,6 @@ async fn session_actor(actor: SessionActor) { &integration, &mut driver, &activity, - &background_jobs.observations, ).await; if let Err(error) = result { eprintln!("autonomous ACP continuation failed for {session_id}: {error}"); @@ -1923,27 +1934,12 @@ fn record_acp_runtime_failure( session_id: &agentkit_acp::SessionId, code: &str, error: impl ToString, -) -> AcpRuntimeError { - record_acp_runtime_failure_observed( - session_id, - code, - error, - &crate::effects::Observations::default(), - ) -} - -fn record_acp_runtime_failure_observed( - session_id: &agentkit_acp::SessionId, - code: &str, - error: impl ToString, - observations: &crate::effects::Observations, ) -> AcpRuntimeError { let rendered = error.to_string(); - match crate::fatal::record_runtime_error_with_effects( + match crate::fatal::record_runtime_error( &session_id.to_string(), crate::fatal::Surface::Acp, code, - observations.snapshot(), ) { Ok(path) => AcpRuntimeError::Loop(format!("{rendered}; fatal log: {}", path.display())), Err(log_error) => { @@ -1956,14 +1952,12 @@ fn record_acp_runtime_failure_observed( fn record_acp_loop_failure( session_id: &agentkit_acp::SessionId, error: &LoopError, - observations: &crate::effects::Observations, ) -> AcpRuntimeError { let rendered = crate::fatal::render_loop_error(error); - match crate::fatal::record_loop_error_with_effects( + match crate::fatal::record_loop_error( &session_id.to_string(), crate::fatal::Surface::Acp, error, - observations.snapshot(), ) { Ok(Some(path)) => { AcpRuntimeError::Loop(format!("{rendered}; fatal log: {}", path.display())) @@ -1997,14 +1991,11 @@ async fn drive_prompt( skill_catalog .submit(skills, items, |items| driver.submit_input(items)) .map_err(|error| match error { - skill_catalog::SubmitError::Catalog(error) => record_acp_runtime_failure_observed( - session_id, - "skill_catalog", - error, - &background_jobs.observations, - ), + skill_catalog::SubmitError::Catalog(error) => { + record_acp_runtime_failure(session_id, "skill_catalog", error) + } skill_catalog::SubmitError::Submit(error) => { - record_acp_loop_failure(session_id, &error, &background_jobs.observations) + record_acp_loop_failure(session_id, &error) } })?; drive_submitted_prompt( @@ -2031,38 +2022,22 @@ async fn drive_runtime_prompt( structured_completion: bool, ) -> Result { if structured_completion { - let _ = settle_background_jobs(tasks, background_jobs) - .await - .map_err(|error| { - record_acp_runtime_failure_observed( - session_id, - "prompt_preparation_settlement", - error, - &background_jobs.observations, - ) - })?; + let _ = settle_background_jobs(tasks, background_jobs).await?; } - let current = runtime.current_skills().await.map_err(|error| { - record_acp_runtime_failure_observed( - session_id, - "skill_refresh", - error, - &background_jobs.observations, - ) - })?; + let current = runtime + .current_skills() + .await + .map_err(AcpRuntimeError::Loop)?; background_jobs.begin_turn(); let items = integration.input_port().prompt_to_items(&request)?; skill_catalog .submit(¤t.skills, items, |items| driver.submit_input(items)) .map_err(|error| match error { - skill_catalog::SubmitError::Catalog(error) => record_acp_runtime_failure_observed( - session_id, - "skill_catalog", - error, - &background_jobs.observations, - ), + skill_catalog::SubmitError::Catalog(error) => { + record_acp_runtime_failure(session_id, "skill_catalog", error) + } skill_catalog::SubmitError::Submit(error) => { - record_acp_loop_failure(session_id, &error, &background_jobs.observations) + record_acp_loop_failure(session_id, &error) } })?; drop(current); @@ -2072,7 +2047,6 @@ async fn drive_runtime_prompt( driver, true, structured_completion.then_some((tasks, background_jobs)), - &background_jobs.observations, ) .await } @@ -2102,14 +2076,14 @@ async fn drive_unsolicited( integration: &AcpIntegration, driver: &mut LoopDriver, activity: &activity::SessionActivity, - observations: &crate::effects::Observations, ) -> Result<(), AcpRuntimeError> { activity .execute( activity::ExecutionOrigin::Autonomous, - drive_finalized(session_id, integration, driver, false, None, observations), + drive_finalized(session_id, integration, driver, false, None), |reason| Some(reason.clone()), ) + .instrument(crate::telemetry::error_spans::operation("acp_autonomous")) .await .map(|_| ()) } @@ -2132,19 +2106,8 @@ async fn drive_until_pause( answer_prompt: bool, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, ) -> Result, AcpRuntimeError> { - let fallback = crate::effects::Observations::local_session(); - let observations = structured - .map(|(_, jobs)| &jobs.observations) - .unwrap_or(&fallback); - let reason = drive_finalized( - session_id, - integration, - driver, - answer_prompt, - structured, - observations, - ) - .await?; + let reason = + drive_finalized(session_id, integration, driver, answer_prompt, structured).await?; if answer_prompt { Ok(Some(PromptResponse::new( agentkit_acp::finish_reason_to_stop_reason(&reason)?, @@ -2160,51 +2123,18 @@ async fn drive_finalized( driver: &mut LoopDriver, answer_prompt: bool, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, - observations: &crate::effects::Observations, ) -> Result { let cancellation = integration.cancellation_handle(session_id)?; let generation = cancellation.generation(); - let mut failure_recorded = false; - let result = drive_domain_until_pause( - session_id, - integration, - driver, - answer_prompt, - structured, - observations, - &mut failure_recorded, - ) - .await; - let result = activity::finalize( + let result = + drive_domain_until_pause(session_id, integration, driver, answer_prompt, structured).await; + activity::finalize( activity::ExecutionOutcome::new(result, cancellation.is_cancelled_since(generation)), structured, integration.flush_session_updates(session_id), - |_, origin| { - if (!failure_recorded || origin == activity::FailureOrigin::Finalization) - && let Err(error) = crate::fatal::record_runtime_error_with_effects( - &session_id.to_string(), - crate::fatal::Surface::Acp, - "session_finalization", - observations.snapshot(), - ) - { - tracing::warn!(%error, "could not record finalization observations"); - } - Ok(()) - }, + |_| Ok(()), ) - .await; - if matches!(result, Ok(FinishReason::Cancelled)) - && let Err(error) = crate::fatal::record_loop_error_with_effects( - &session_id.to_string(), - crate::fatal::Surface::Acp, - &LoopError::Cancelled, - observations.snapshot(), - ) - { - tracing::warn!(%error, "could not record cancellation observations"); - } - result + .await } async fn drive_domain_until_pause( @@ -2213,8 +2143,6 @@ async fn drive_domain_until_pause( driver: &mut LoopDriver, answer_prompt: bool, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, - observations: &crate::effects::Observations, - failure_recorded: &mut bool, ) -> Result { let cancellation = integration.cancellation_handle(session_id)?; let generation = cancellation.generation(); @@ -2224,16 +2152,13 @@ async fn drive_domain_until_pause( Err(LoopError::Cancelled) => { return Ok(FinishReason::Cancelled); } - Err(error) => { - *failure_recorded = true; - return Err(record_acp_loop_failure(session_id, &error, observations)); - } + Err(error) => return Err(record_acp_loop_failure(session_id, &error)), }; if cancellation.is_cancelled_since(generation) { - driver.retire_interrupted_turn().await.map_err(|error| { - *failure_recorded = true; - record_acp_loop_failure(session_id, &error, observations) - })?; + driver + .retire_interrupted_turn() + .await + .map_err(|error| record_acp_loop_failure(session_id, &error))?; return Ok(FinishReason::Cancelled); } match step { @@ -2810,7 +2735,7 @@ pub(super) mod tests { ), None, content.flush(), - |_, _| Ok(()), + |_| Ok(()), ) .await }, @@ -4088,11 +4013,6 @@ pub(super) mod tests { #[tokio::test] async fn live_prompt_boundary_refreshes_plugin_skill_catalog() { - if crate::effects::isolated_test( - "protocols::acp::tests::live_prompt_boundary_refreshes_plugin_skill_catalog", - ) { - return; - } let root = tempfile::tempdir().unwrap(); let config = root.path().join("config.toml"); std::fs::write(&config, "").unwrap(); @@ -4184,7 +4104,6 @@ pub(super) mod tests { .unwrap(); let task_manager = AsyncTaskManager::new(); let tasks = task_manager.handle(); - let background_jobs = BackgroundJobs::default(); let response = drive_runtime_prompt( &acp_session_id, &runtime, @@ -4198,52 +4117,13 @@ pub(super) mod tests { )], ), &tasks, - &background_jobs, + &BackgroundJobs::default(), false, ) .await .unwrap(); assert_eq!(response, FinishReason::Completed); assert_eq!(notification_items_seen.load(Ordering::SeqCst), 1); - // Retained facts from earlier work must survive next-prompt preparation failure. - background_jobs.observations.invocation_started(); - std::fs::write(&config, "invalid = [").unwrap(); - let error = drive_runtime_prompt( - &acp_session_id, - &runtime, - &integration, - &mut skill_catalog, - &mut driver, - PromptRequest::new( - acp_session_id.clone(), - vec![agentkit_acp::ContentBlock::Text( - agentkit_acp::TextContent::new("next prompt"), - )], - ), - &tasks, - &background_jobs, - false, - ) - .await - .unwrap_err(); - assert_eq!(error.to_string().matches("fatal log:").count(), 1); - let record = crate::effects::test_record(&acp_session_id.to_string()); - assert_eq!(record["code"], "skill_refresh"); - assert_eq!(record["possible_effects"]["source"], "local_session"); - assert_eq!( - record["possible_effects"]["tool_execution_start_reported"], - true - ); - assert_eq!( - std::fs::read_dir( - std::path::PathBuf::from(std::env::var_os("HOME").unwrap()) - .join(".kit/errors") - .join(acp_session_id.to_string()) - ) - .unwrap() - .count(), - 1 - ); drain.abort(); } @@ -4422,15 +4302,9 @@ pub(super) mod tests { .unwrap(); for _ in 0..2 { - drive_unsolicited( - &session_id, - &integration, - &mut driver, - &activity, - &crate::effects::Observations::local_session(), - ) - .await - .unwrap(); + drive_unsolicited(&session_id, &integration, &mut driver, &activity) + .await + .unwrap(); } assert_eq!(turns.load(Ordering::SeqCst), 2); assert!(states.try_recv().is_err()); @@ -4438,30 +4312,18 @@ pub(super) mod tests { driver .submit_input(vec![Item::notification("background result")]) .unwrap(); - drive_unsolicited( - &session_id, - &integration, - &mut driver, - &activity, - &crate::effects::Observations::local_session(), - ) - .await - .unwrap(); + drive_unsolicited(&session_id, &integration, &mut driver, &activity) + .await + .unwrap(); let started = states.try_recv().unwrap(); let ended = states.try_recv().unwrap(); assert!(started.active && !ended.active); assert_eq!(started.turn_id, 1); assert_eq!(started.turn_id, ended.turn_id); assert!(ended.error.is_none()); - drive_unsolicited( - &session_id, - &integration, - &mut driver, - &activity, - &crate::effects::Observations::local_session(), - ) - .await - .unwrap(); + drive_unsolicited(&session_id, &integration, &mut driver, &activity) + .await + .unwrap(); assert_eq!(turns.load(Ordering::SeqCst), 3); assert!(states.try_recv().is_err()); @@ -4895,108 +4757,6 @@ pub(super) mod tests { } } - #[tokio::test] - async fn startup_failure_records_once_with_original_classification_and_source() { - if crate::effects::isolated_test( - "protocols::acp::tests::startup_failure_records_once_with_original_classification_and_source", - ) { - return; - } - for before_owner in [false, true] { - let root = tempfile::tempdir().unwrap(); - let other_root = tempfile::tempdir().unwrap(); - let session_id = crate::session::new_id(); - let runtime = Runtime::with_session_provider_credentials_effort_and_openrouter_key( - root.path(), - "test/model", - crate::ProviderKind::OpenRouter, - crate::runtime::SessionRequest { - id: session_id.clone(), - resume: false, - force: false, - }, - crate::credentials::CredentialStorage::Memory, - None, - // SelectableAdapter accepts the selection, but starting its - // model session fails deterministically without network I/O. - Some(crate::provider::OpenRouterApiKey::new("")), - ) - .unwrap(); - let workspace = if before_owner { - other_root.path() - } else { - root.path() - } - .to_path_buf(); - let (client_transport, agent_transport) = Channel::duplex(); - let server = tokio::spawn(serve_transport(runtime, agent_transport)); - agent_client_protocol::Client - .builder() - .connect_with(client_transport, async move |connection| { - connection - .send_request(InitializeRequest::new(ProtocolVersion::V1)) - .block_task() - .await?; - let error = connection - .send_request(NewSessionRequest::new(workspace)) - .block_task() - .await - .expect_err("startup must fail"); - let rendered = error.to_string(); - assert_eq!(rendered.matches("fatal log:").count(), 1, "{rendered}"); - assert!( - rendered.contains(if before_owner { - "this Kit runtime is fixed to" - } else { - "--openrouter-api-key cannot be empty" - }), - "{rendered}" - ); - Ok(()) - }) - .await - .unwrap(); - server.abort(); - let _ = server.await; - - let directory = PathBuf::from(std::env::var_os("HOME").unwrap()) - .join(".kit/errors") - .join(&session_id); - let records: Vec<_> = std::fs::read_dir(directory) - .unwrap() - .map(|entry| entry.unwrap().path()) - .filter(|path| path.extension().is_some_and(|ext| ext == "json")) - .collect(); - assert_eq!(records.len(), 1, "startup must record exactly once"); - let record: serde_json::Value = - serde_json::from_slice(&std::fs::read(&records[0]).unwrap()).unwrap(); - assert_eq!(record["surface"], "acp"); - assert_eq!(record["kind"], "runtime"); - assert_eq!( - record["code"], - if before_owner { - "session_start" - } else { - "invalid_state" - } - ); - assert_eq!( - record["possible_effects"]["source"], - if before_owner { - "unknown" - } else { - "local_session" - } - ); - assert_eq!(record["possible_effects"]["observation_incomplete"], true); - assert_eq!( - record["possible_effects"]["tool_execution_start_reported"], - false - ); - assert!(crate::session::load(root.path(), &session_id).is_err()); - } - } - #[tokio::test] async fn registry_shutdown_closes_real_session_and_rejects_reattach() { let root = tempfile::tempdir().unwrap(); @@ -5301,89 +5061,4 @@ pub(super) mod tests { server.abort(); let _ = server.await; } - #[tokio::test] - async fn root_cancellation_keeps_owner_in_unstructured_and_unsolicited_drives() { - if crate::effects::isolated_test( - "protocols::acp::tests::root_cancellation_keeps_owner_in_unstructured_and_unsolicited_drives", - ) { - return; - } - for autonomous in [false, true] { - let session_id = agentkit_acp::SessionId::new(if autonomous { - "effects-autonomous" - } else { - "effects-prompt" - }); - let loop_id = AgentkitSessionId::new(session_id.to_string()); - let integration = AcpIntegration::builder() - .name("effects-test") - .approval_resolver(AutoDenyResolver) - .build() - .unwrap(); - let (client, mut messages) = AcpClientHandle::channel(); - integration - .bind_session(AcpSessionBinding::new( - session_id.clone(), - loop_id.clone(), - client, - )) - .unwrap(); - let drain = tokio::spawn(async move { - while let Some(message) = messages.recv().await { - if let AcpClientMessage::Flush { response } = message { - let _ = response.send(()); - } - } - }); - let observations = crate::effects::Observations::local_session(); - // Evidence from a local background invocation in this live session. - observations.invocation_started(); - let mut driver = Agent::builder() - .model(CancelAdapter) - .observer(observations.clone()) - .input(vec![Item::text(ItemKind::User, "cancel")]) - .build() - .unwrap() - .start(SessionConfig::new(loop_id).without_cache()) - .await - .unwrap(); - if autonomous { - let (notifications, _) = mpsc::unbounded_channel(); - let activity = test_activity(session_id.clone(), notifications); - drive_unsolicited( - &session_id, - &integration, - &mut driver, - &activity, - &observations, - ) - .await - .unwrap(); - } else { - let reason = drive_finalized( - &session_id, - &integration, - &mut driver, - true, - None, - &observations, - ) - .await - .unwrap(); - assert_eq!(reason, FinishReason::Cancelled); - } - let record = crate::effects::test_record(&session_id.to_string()); - assert_eq!(record["kind"], "cancelled"); - assert_eq!(record["possible_effects"]["source"], "local_session"); - assert_eq!( - record["possible_effects"]["tool_execution_start_reported"], - true - ); - assert_eq!( - record["possible_effects"]["tool_execution_completion_reported"], - false - ); - drain.abort(); - } - } } diff --git a/src/protocols/acp/activity.rs b/src/protocols/acp/activity.rs index 1dce072c..e7672a3e 100644 --- a/src/protocols/acp/activity.rs +++ b/src/protocols/acp/activity.rs @@ -164,13 +164,6 @@ impl ExecutionOutcome { } } -/// Identifies whether finalization replaced the execution's original error. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum FailureOrigin { - Execution, - Finalization, -} - /// Shared finalization order: cancel/drain structured work, drain all content even /// on failure, render a diagnostic, then return the outcome for single settlement. /// Hooks are transport only; neither hook chooses lifecycle or cleanup policy. @@ -181,25 +174,22 @@ pub(super) async fn finalize( &crate::runtime::BackgroundJobs, )>, flush: impl std::future::Future>, - diagnostic: impl FnOnce(&AcpRuntimeError, FailureOrigin) -> Result<(), AcpRuntimeError>, + diagnostic: impl FnOnce(&AcpRuntimeError) -> Result<(), AcpRuntimeError>, ) -> Result { let mut result = outcome.result; - let mut origin = FailureOrigin::Execution; if (result.is_err() || matches!(result, Ok(FinishReason::Cancelled))) && let Some((tasks, jobs)) = structured { super::cancel_background_jobs(tasks, jobs).await; if let Err(error) = super::settle_background_jobs(tasks, jobs).await { result = Err(error); - origin = FailureOrigin::Finalization; } } if let Err(error) = flush.await { result = Err(error); - origin = FailureOrigin::Finalization; } if let Err(error) = &result { - diagnostic(error, origin)?; + diagnostic(error)?; } result } @@ -306,7 +296,7 @@ mod tests { ExecutionOutcome::new(Err(AcpRuntimeError::Loop("provider failed".into())), true), None, async { Ok(()) }, - |_, _| panic!("cancelled model failure is not an error"), + |_| panic!("cancelled model failure is not an error"), ) .await .unwrap(); @@ -333,8 +323,7 @@ mod tests { order.lock().unwrap().push("flush"); Err(AcpRuntimeError::ClientClosed) }, - |error, origin| { - assert_eq!(origin, FailureOrigin::Finalization); + |error| { order.lock().unwrap().push("diagnostic"); diagnostic.lock().unwrap().push(error.to_string()); Ok(()) @@ -354,37 +343,6 @@ mod tests { assert_eq!(diagnostic.lock().unwrap().len(), 1); } - #[tokio::test] - async fn finalization_marks_replacement_of_an_already_recorded_failure() { - for flush_fails in [false, true] { - let result = finalize( - ExecutionOutcome::new(Err(AcpRuntimeError::Loop("driver failed".into())), false), - None, - async { - if flush_fails { - Err(AcpRuntimeError::ClientClosed) - } else { - Ok(()) - } - }, - |error, origin| { - assert_eq!( - origin, - if flush_fails { - FailureOrigin::Finalization - } else { - FailureOrigin::Execution - } - ); - assert_eq!(matches!(error, AcpRuntimeError::ClientClosed), flush_fails); - Ok(()) - }, - ) - .await; - assert!(result.is_err()); - } - } - #[tokio::test] async fn failed_terminal_projection_is_not_retried() { let calls = Arc::new(Mutex::new(0)); diff --git a/src/protocols/acp/v2.rs b/src/protocols/acp/v2.rs index 563f9a32..548f8949 100644 --- a/src/protocols/acp/v2.rs +++ b/src/protocols/acp/v2.rs @@ -26,6 +26,7 @@ use agentkit_loop::{ use agentkit_task_manager::{TaskEvent, TaskManagerHandle}; use async_trait::async_trait; use tokio::sync::{mpsc, oneshot, watch}; +use tracing::Instrument as _; use crate::{ provider::{ProviderKind, SelectableAdapter, authentication_method_id}, @@ -119,40 +120,23 @@ fn list_sessions_error(error: ListSessionsError) -> agent_client_protocol::Error } } -#[cfg(test)] fn map_loop_error(session_id: &wire::SessionId, error: &LoopError) -> AcpRuntimeError { - map_loop_error_observed(session_id, error, &crate::effects::Observations::default()) -} -#[cfg(test)] -fn loop_error_stop_reason( - session_id: &wire::SessionId, - error: &LoopError, -) -> Result { - loop_error_stop_reason_observed(session_id, error, &crate::effects::Observations::default()) -} - -fn map_loop_error_observed( - session_id: &wire::SessionId, - error: &LoopError, - observations: &crate::effects::Observations, -) -> AcpRuntimeError { if matches!(error, LoopError::Cancelled) { AcpRuntimeError::Cancelled } else { let session_id = agentkit_acp::SessionId::new(session_id.to_string()); - super::record_acp_loop_failure(&session_id, error, observations) + super::record_acp_loop_failure(&session_id, error) } } -fn loop_error_stop_reason_observed( +fn loop_error_stop_reason( session_id: &wire::SessionId, error: &LoopError, - observations: &crate::effects::Observations, ) -> Result { if matches!(error, LoopError::Cancelled) { Ok(FinishReason::Cancelled) } else { - Err(map_loop_error_observed(session_id, error, observations)) + Err(map_loop_error(session_id, error)) } } @@ -1177,6 +1161,7 @@ async fn session_actor(actor: SessionActor) &background_jobs, structured_completion, &activity,) + .instrument(crate::telemetry::error_spans::operation("acp")) .await; busy.store(false, Ordering::Release); if let Err(error) = result { @@ -1202,21 +1187,22 @@ async fn session_actor(actor: SessionActor) }, event = mcp_events.recv() => { if let Some(event) = event { - let result = match driver.submit_input(vec![Item::notification(event.message)]) { - Ok(()) => drive_autonomous( - &session_id, - &integration, - &handle, - &busy, - &mut driver, - &sink, - SessionInstruments { - activity: &activity, - observations: &background_jobs.observations, - }, - ).await, - Err(error) => Err(map_loop_error_observed(&session_id, &error, &background_jobs.observations)), - }; + let result = async { + match driver.submit_input(vec![Item::notification(event.message)]) { + Ok(()) => drive_autonomous( + &session_id, + &integration, + &handle, + &busy, + &mut driver, + &sink, + &activity, + ).await, + Err(error) => Err(map_loop_error(&session_id, &error)), + } + } + .instrument(crate::telemetry::error_spans::operation("acp_autonomous")) + .await; if let Err(error) = result { eprintln!("ACP v2 autonomous turn failed for {session_id}: {error}"); } @@ -1233,11 +1219,9 @@ async fn session_actor(actor: SessionActor) &busy, &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &background_jobs.observations, - }, - ).await + &activity,) + .instrument(crate::telemetry::error_spans::operation("acp_autonomous")) + .await { eprintln!("ACP v2 autonomous turn failed for {session_id}: {error}"); } @@ -1282,12 +1266,6 @@ async fn prepare_prompt( && let Err(error) = super::settle_background_jobs(tasks, background_jobs).await { handle.stop_injection_turn(); - let error = super::record_acp_runtime_failure_observed( - &agentkit_acp::SessionId::new(session_id.to_string()), - "prompt_preparation_settlement", - error, - &background_jobs.observations, - ); let _ = reply.send(Err(error)); return Ok(()); } @@ -1300,13 +1278,7 @@ async fn prepare_prompt( Ok(current) => current, Err(error) => { handle.stop_injection_turn(); - let error = super::record_acp_runtime_failure_observed( - &agentkit_acp::SessionId::new(session_id.to_string()), - "skill_refresh", - error, - &background_jobs.observations, - ); - let _ = reply.send(Err(error)); + let _ = reply.send(Err(AcpRuntimeError::Loop(error))); return Ok(()); } }; @@ -1319,16 +1291,9 @@ async fn prepare_prompt( .submit(skills, items, |items| driver.submit_input(items)) .map_err(|error| match error { skill_catalog::SubmitError::Catalog(error) => { - super::record_acp_runtime_failure_observed( - &agentkit_acp::SessionId::new(session_id.to_string()), - "skill_catalog", - error, - &background_jobs.observations, - ) - } - skill_catalog::SubmitError::Submit(error) => { - map_loop_error_observed(session_id, &error, &background_jobs.observations) + AcpRuntimeError::Loop(format!("skill catalog error: {error}")) } + skill_catalog::SubmitError::Submit(error) => map_loop_error(session_id, &error), })?; integration.begin_prompt(session_id) }); @@ -1365,7 +1330,6 @@ async fn prepare_prompt( structured_completion.then_some((tasks, background_jobs)), activity, ExecutionOrigin::Prompt, - &background_jobs.observations, ) .await } @@ -1405,7 +1369,6 @@ impl TurnControl for AcpSessionHandle { } } -#[cfg(test)] async fn drive_prompt( session_id: &wire::SessionId, driver: &mut LoopDriver, @@ -1413,33 +1376,6 @@ async fn drive_prompt( cancellation_generation: u64, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, ) -> Result -where - S: ModelSession + Send + 'static, - C: TurnControl, -{ - let observations = crate::effects::Observations::local_session(); - let mut recorded = false; - drive_prompt_observed( - session_id, - driver, - control, - cancellation_generation, - structured, - &observations, - &mut recorded, - ) - .await -} - -async fn drive_prompt_observed( - session_id: &wire::SessionId, - driver: &mut LoopDriver, - control: &C, - cancellation_generation: u64, - structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, - observations: &crate::effects::Observations, - failure_recorded: &mut bool, -) -> Result where S: ModelSession + Send + 'static, C: TurnControl, @@ -1450,8 +1386,6 @@ where control, cancellation_generation, structured, - observations, - failure_recorded, ) .await; if matches!(result, Ok(FinishReason::Cancelled)) { @@ -1471,8 +1405,6 @@ async fn drive_prompt_inner( control: &C, cancellation_generation: u64, structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, - observations: &crate::effects::Observations, - failure_recorded: &mut bool, ) -> Result where S: ModelSession + Send + 'static, @@ -1486,8 +1418,7 @@ where if control.is_cancelled_since(cancellation_generation) { return Ok(FinishReason::Cancelled); } - *failure_recorded = !matches!(error, LoopError::Cancelled); - return loop_error_stop_reason_observed(session_id, &error, observations); + return loop_error_stop_reason(session_id, &error); } }; if control.is_cancelled_since(cancellation_generation) { @@ -1575,8 +1506,7 @@ where if control.is_cancelled_since(cancellation_generation) { return Ok(FinishReason::Cancelled); } - *failure_recorded = !matches!(error, LoopError::Cancelled); - return loop_error_stop_reason_observed(session_id, &error, observations); + return loop_error_stop_reason(session_id, &error); } } } @@ -1594,21 +1524,17 @@ async fn run_active_turn( structured: Option<(&TaskManagerHandle, &BackgroundJobs)>, activity: &SessionActivity, origin: ExecutionOrigin, - observations: &crate::effects::Observations, ) -> Result<(), AcpRuntimeError> { activity .execute( origin, async { - let mut failure_recorded = false; - let result = drive_prompt_observed( + let result = drive_prompt( session_id, driver, handle, cancellation_generation, structured, - observations, - &mut failure_recorded, ) .await; let outcome = super::activity::ExecutionOutcome::new( @@ -1622,31 +1548,9 @@ async fn run_active_turn( outcome, structured, integration.flush_session_updates(session_id), - |error, origin| { - if (!failure_recorded || origin == super::activity::FailureOrigin::Finalization) - && let Err(log_error) = crate::fatal::record_runtime_error_with_effects( - &session_id.to_string(), - crate::fatal::Surface::Acp, - "session_finalization", - observations.snapshot(), - ) - { - tracing::warn!(%log_error, "could not record v2 finalization observations"); - } - sink.update(error_diagnostic_notification(session_id, error)) - }, + |error| sink.update(error_diagnostic_notification(session_id, error)), ) .await; - if matches!(result, Ok(FinishReason::Cancelled)) - && let Err(error) = crate::fatal::record_loop_error_with_effects( - &session_id.to_string(), - crate::fatal::Surface::Acp, - &LoopError::Cancelled, - observations.snapshot(), - ) - { - tracing::warn!(%error, "could not record v2 cancellation observations"); - } integration.finish_prompt(session_id); result }, @@ -1656,12 +1560,6 @@ async fn run_active_turn( .map(|_| ()) } -/// Session-owned lifecycle and effects instruments used during execution. -struct SessionInstruments<'a> { - activity: &'a SessionActivity, - observations: &'a crate::effects::Observations, -} - async fn drive_autonomous( session_id: &wire::SessionId, integration: &AcpIntegration, @@ -1669,7 +1567,7 @@ async fn drive_autonomous( busy: &AtomicBool, driver: &mut LoopDriver, sink: &ResponseReplacementSink, - instruments: SessionInstruments<'_>, + activity: &SessionActivity, ) -> Result<(), AcpRuntimeError> { if claim_prompt(busy).is_err() { return Ok(()); @@ -1686,9 +1584,8 @@ async fn drive_autonomous( sink, cancellation_generation, None, - instruments.activity, + activity, ExecutionOrigin::Autonomous, - instruments.observations, ) .await; integration.finish_prompt(session_id); @@ -3246,7 +3143,6 @@ mod tests { handle.start_injection_turn(); let generation = handle.cancellation_handle().generation(); let background_jobs = BackgroundJobs::default(); - let observations = crate::effects::Observations::local_session(); let prompt = run_active_turn( &session_id, &integration, @@ -3257,7 +3153,6 @@ mod tests { Some((&tasks, &background_jobs)), &activity, ExecutionOrigin::Prompt, - &observations, ); tokio::pin!(prompt); @@ -3574,10 +3469,7 @@ mod tests { &busy, &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &crate::effects::Observations::local_session(), - }, + &activity, ) .await .unwrap(); @@ -3634,10 +3526,7 @@ mod tests { &busy, &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &crate::effects::Observations::local_session(), - }, + &activity, ) .await .unwrap(); @@ -3654,10 +3543,7 @@ mod tests { &busy, &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &crate::effects::Observations::local_session(), - }, + &activity, ) .await .unwrap(); @@ -3723,10 +3609,7 @@ mod tests { &busy, &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &crate::effects::Observations::local_session(), - }, + &activity, ) .await; @@ -3800,10 +3683,7 @@ mod tests { &busy, &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &crate::effects::Observations::local_session(), - }, + &activity, ) .await .unwrap(); @@ -3860,10 +3740,7 @@ mod tests { &AtomicBool::new(false), &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &crate::effects::Observations::local_session(), - }, + &activity, ) .await; assert!(matches!(result, Err(AcpRuntimeError::ClientClosed))); @@ -3932,10 +3809,7 @@ mod tests { &busy, &mut driver, &sink, - SessionInstruments { - activity: &activity, - observations: &crate::effects::Observations::local_session(), - }, + &activity, ) .await; @@ -4512,199 +4386,4 @@ mod tests { Err(ListSessionsError::InvalidCursor) )); } - #[tokio::test] - async fn root_stream_cancellation_records_the_same_local_owner() { - if crate::effects::isolated_test( - "protocols::acp::v2::tests::root_stream_cancellation_records_the_same_local_owner", - ) { - return; - } - let integration = AcpIntegration::default(); - let recording = RecordingSink::default(); - let sink = ResponseReplacementSink::new(recording.clone()); - let session_id = wire::SessionId::new("effects-v2-cancel"); - let loop_id = SessionId::new("effects-v2-loop"); - let activity = native_activity(session_id.clone(), sink.clone()); - let handle = integration - .bind_session( - AcpSessionBinding::new(session_id.clone(), loop_id.clone(), sink.clone()) - .cancellation(CancellationController::new()), - ) - .unwrap(); - let observations = crate::effects::Observations::local_session(); - let observer = ResponseReplacementObserver::new( - integration.clone(), - sink.clone(), - session_id.clone(), - activity.clone(), - ); - let mut driver = Agent::builder() - .model(StreamingCancellationAdapter { - interrupt: handle.clone(), - }) - .observer(observer) - .observer(observations.clone()) - .cancellation(handle.cancellation_handle()) - .build() - .unwrap() - .start(SessionConfig::new(loop_id).without_cache()) - .await - .unwrap(); - driver - .submit_input(vec![Item::text(ItemKind::User, "private prompt")]) - .unwrap(); - handle.prepare_injection_turn(); - let generation = handle.cancellation_handle().generation(); - handle.start_injection_turn(); - run_active_turn( - &session_id, - &integration, - &handle, - &mut driver, - &sink, - generation, - None, - &activity, - ExecutionOrigin::Prompt, - &observations, - ) - .await - .unwrap(); - let record = crate::effects::test_record(&session_id.to_string()); - assert_eq!(record["kind"], "cancelled"); - assert_eq!(record["possible_effects"]["source"], "local_session"); - assert_eq!( - record["possible_effects"]["assistant_output_observed"], - true - ); - assert_eq!( - record["possible_effects"]["tool_execution_start_reported"], - false - ); - assert!( - !serde_json::to_string(&record) - .unwrap() - .contains("private prompt") - ); - assert_running_then_idle( - &recording.updates.lock().unwrap(), - wire::StopReason::Cancelled, - ); - } - #[tokio::test] - async fn preparation_failure_retains_previous_prompt_observations() { - if crate::effects::isolated_test( - "protocols::acp::v2::tests::preparation_failure_retains_previous_prompt_observations", - ) { - return; - } - let root = tempfile::tempdir().unwrap(); - let config = root.path().join("config.toml"); - std::fs::write(&config, "").unwrap(); - let plugins = crate::plugins::PluginRuntime::load( - config.clone(), - root.path().to_path_buf(), - root.path().join("cache"), - root.path().join("data"), - ) - .await - .unwrap(); - let runtime = Runtime::with_plugin_runtime( - Runtime::new(root.path(), "gpt-5.4").unwrap(), - Some(plugins), - ) - .unwrap(); - let runtime = Runtime::with_mcp_config( - runtime, - None, - Vec::new(), - false, - crate::tools::mcp::CredentialStorage::Memory, - ) - .await - .unwrap(); - let baseline = runtime.current_skills().await.unwrap(); - let mut skill_catalog = skill_catalog::SkillCatalogMonitor::new(&baseline.skills).unwrap(); - drop(baseline); - let integration = AcpIntegration::default(); - let recording = RecordingSink::default(); - let sink = ResponseReplacementSink::new(recording.clone()); - let session_id = wire::SessionId::new("effects-v2-preparation"); - let loop_id = SessionId::new("effects-v2-preparation-loop"); - let activity = native_activity(session_id.clone(), sink.clone()); - let handle = integration - .bind_session(AcpSessionBinding::new( - session_id.clone(), - loop_id.clone(), - sink.clone(), - )) - .unwrap(); - let turns = Arc::new(AtomicU64::new(0)); - let mut driver = Agent::builder() - .model(TestAdapter { - outcome: TestOutcome::ProviderError, - turns: turns.clone(), - interrupt: None, - }) - .build() - .unwrap() - .start(SessionConfig::new(loop_id).without_cache()) - .await - .unwrap(); - let manager = AsyncTaskManager::new(); - let tasks = manager.handle(); - let jobs = BackgroundJobs::default(); - // A prior invocation's observations are cumulative, not attributed to this rejected prompt. - jobs.observations.invocation_started(); - jobs.begin_turn(); - std::fs::write(&config, "invalid = [").unwrap(); - handle.prepare_injection_turn(); - let (reply, response) = oneshot::channel(); - prepare_prompt( - &session_id, - PromptSkillSource::Runtime(&runtime), - &integration, - &handle, - &mut skill_catalog, - &mut driver, - PromptCommand { - request: wire::PromptRequest::new( - session_id.clone(), - vec![wire::ContentBlock::Text(wire::TextContent::new( - "next prompt", - ))], - ), - cancellation_generation: handle.cancellation_handle().generation(), - reply, - }, - &sink, - &tasks, - &jobs, - false, - &activity, - ) - .await - .unwrap(); - let error = response.await.unwrap().unwrap_err(); - assert_eq!(error.to_string().matches("fatal log:").count(), 1); - assert_eq!(turns.load(Ordering::Relaxed), 0); - assert!(recording.updates.lock().unwrap().is_empty()); - let record = crate::effects::test_record(&session_id.to_string()); - assert_eq!(record["code"], "skill_refresh"); - assert_eq!(record["possible_effects"]["source"], "local_session"); - assert_eq!( - record["possible_effects"]["tool_execution_start_reported"], - true - ); - assert_eq!( - std::fs::read_dir( - std::path::PathBuf::from(std::env::var_os("HOME").unwrap()) - .join(".kit/errors") - .join(session_id.to_string()) - ) - .unwrap() - .count(), - 1 - ); - } } diff --git a/src/runtime.rs b/src/runtime.rs index 9a0be759..5e9ea4c3 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -30,6 +30,7 @@ use async_trait::async_trait; use serde_json::{Value, json}; use tokio::sync::watch; use tokio_util::sync::CancellationToken; +use tracing::Instrument as _; use crate::{ acp_child::{AcpHarnesses, BUILTIN_HARNESS, ChildConfig}, @@ -266,19 +267,6 @@ impl Drop for SessionClaim { } } -// Preserve startup classification until the single recording boundary, including -// failures that occur before a local observation owner has been created. -enum AcpStartupFailure { - Runtime(AcpRuntimeError), - Loop(LoopError, crate::effects::Observations), -} - -impl From for AcpStartupFailure { - fn from(error: AcpRuntimeError) -> Self { - Self::Runtime(error) - } -} - pub(crate) struct AcpDriver { pub driver: LoopDriver, pub skills: Vec, @@ -964,83 +952,40 @@ impl Runtime { skills: Arc, ) -> ComposeOnly { let mut children = agentkit_tools_core::ToolRegistry::new() - .with( - Observed::new(ArtifactTool::new(crate::artifacts::base(&self.root))) - .with_observations(background_jobs.observations.clone()), - ) - .with( - Observed::new(DocsTool::new()) - .with_observations(background_jobs.observations.clone()), - ) - .with( - Observed::new(ShellTool::new(self.root.clone())) - .with_observations(background_jobs.observations.clone()), - ) - .with( - Observed::new(EditTool::new(self.root.clone())) - .with_observations(background_jobs.observations.clone()), - ); + .with(Observed::new(ArtifactTool::new(crate::artifacts::base( + &self.root, + )))) + .with(Observed::new(DocsTool::new())) + .with(Observed::new(ShellTool::new(self.root.clone()))) + .with(Observed::new(EditTool::new(self.root.clone()))); if depth < self.max_subagent_depth { children - .register( - Observed::new(SubagentTool::new(subagents.clone(), depth)) - .with_observations(background_jobs.observations.clone()), - ) - .register( - Observed::new(ForkTool::new(subagents.clone(), depth)) - .with_observations(background_jobs.observations.clone()), - ); + .register(Observed::new(SubagentTool::new(subagents.clone(), depth))) + .register(Observed::new(ForkTool::new(subagents.clone(), depth))); } children - .register( - Observed::new(PromptTool::new(subagents.clone())) - .with_observations(background_jobs.observations.clone()), - ) - .register( - Observed::new(SubagentsTool::new(subagents.clone())) - .with_observations(background_jobs.observations.clone()), - ) - .register( - Observed::new(CloseTool::new(subagents, { - let background_jobs = background_jobs.clone(); - move |call_id, allow_pending| { - if allow_pending { - background_jobs.cancel(call_id) - } else { - background_jobs.cancel_running(call_id) - } + .register(Observed::new(PromptTool::new(subagents.clone()))) + .register(Observed::new(SubagentsTool::new(subagents.clone()))) + .register(Observed::new(CloseTool::new(subagents, { + let background_jobs = background_jobs.clone(); + move |call_id, allow_pending| { + if allow_pending { + background_jobs.cancel(call_id) + } else { + background_jobs.cancel_running(call_id) } - })) - .with_observations(background_jobs.observations.clone()), - ) - .register( - Observed::new(A2aTool::new()) - .with_observations(background_jobs.observations.clone()), - ) - .register( - Observed::new(ToolSearch::new(self.mcp.clone())) - .with_observations(background_jobs.observations.clone()), - ) - .register( - Observed::new(AuthTool::new(self.mcp.clone())) - .with_observations(background_jobs.observations.clone()), - ) - .register( - Observed::new(McpTool::new(self.mcp.clone())) - .with_observations(background_jobs.observations.clone()), - ); + } + }))) + .register(Observed::new(A2aTool::new())) + .register(Observed::new(ToolSearch::new(self.mcp.clone()))) + .register(Observed::new(AuthTool::new(self.mcp.clone()))) + .register(Observed::new(McpTool::new(self.mcp.clone()))); if let Some(skill_tool) = &self.dynamic_skill_tool { - children.register(observe_shared( - Arc::clone(skill_tool), - background_jobs.observations.clone(), - )); + children.register(observe_shared(Arc::clone(skill_tool))); } else { let skill_tools = skills.tool_registry(); if let Some(skill_tool) = skill_tools.get(&ToolName::new("skill")) { - children.register(observe_shared( - skill_tool, - background_jobs.observations.clone(), - )); + children.register(observe_shared(skill_tool)); } } let hidden_tools = children.clone(); @@ -1063,11 +1008,20 @@ impl Runtime { } pub async fn run(self: &Arc, prompt: String, depth: usize) -> Result { - self.run_interruptible(prompt, depth, None).await + self.run_interruptible(prompt, depth, None) + .instrument(crate::telemetry::error_spans::operation("prompt")) + .await } /// Runs one prompt in the configured durable session. pub async fn run_persistent(self: &Arc, prompt: String) -> Result { + // Keep the operation current through startup and the existing fatal writes. + self.run_persistent_inner(prompt) + .instrument(crate::telemetry::error_spans::operation("prompt")) + .await + } + + async fn run_persistent_inner(self: &Arc, prompt: String) -> Result { let request = self .session .lock() @@ -1128,25 +1082,27 @@ impl Runtime { ) })?; let subagents = self.subagents.fresh(); - let background_jobs = BackgroundJobs::default(); let agent = Agent::builder() .model(self.adapter.clone()) .telemetry(self.agentkit_telemetry()) - .add_tool_source(self.compose_with_jobs(0, subagents, background_jobs.clone(), skills)) + .add_tool_source(self.compose_with_jobs( + 0, + subagents, + BackgroundJobs::default(), + skills, + )) .task_manager(background_task_manager()) .mutator(compactor) - .observer(background_jobs.observations.clone()) .transcript_observer(opened.observer) .transcript(opened.transcript) .input(vec![Item::text(ItemKind::User, prompt)]) .build() .map_err(|error| { - record_runtime_failure_observed( + record_runtime_failure( &session_id, crate::fatal::Surface::Prompt, "agent_build", error.to_string(), - &background_jobs.observations, ) })?; let mut driver = match agent @@ -1164,7 +1120,6 @@ impl Runtime { &session_id, crate::fatal::Surface::Prompt, &error, - &background_jobs.observations, )); } }; @@ -1174,7 +1129,6 @@ impl Runtime { &session_id, crate::fatal::Surface::Prompt, &error, - &background_jobs.observations, )), } } @@ -1184,22 +1138,6 @@ impl Runtime { prompt: String, depth: usize, cancellation: Option, - ) -> Result { - self.run_cancelled_observed( - prompt, - depth, - cancellation, - crate::effects::Observations::local_session(), - ) - .await - } - - pub(crate) async fn run_cancelled_observed( - self: &Arc, - prompt: String, - depth: usize, - cancellation: Option, - observations: crate::effects::Observations, ) -> Result { let controller = CancellationController::new(); let bridge = cancellation.map(|token| { @@ -1210,7 +1148,7 @@ impl Runtime { }) }); let result = self - .run_interruptible_observed(prompt, depth, Some(controller.handle()), observations) + .run_interruptible(prompt, depth, Some(controller.handle())) .await; if let Some(bridge) = bridge { bridge.abort(); @@ -1225,22 +1163,6 @@ impl Runtime { prompt: String, depth: usize, cancellation: Option, - ) -> Result { - self.run_interruptible_observed( - prompt, - depth, - cancellation, - crate::effects::Observations::local_session(), - ) - .await - } - - async fn run_interruptible_observed( - self: &Arc, - prompt: String, - depth: usize, - cancellation: Option, - observations: crate::effects::Observations, ) -> Result { if crate::resilient_fs::shutdown_token().is_cancelled() { return Err(LoopError::InvalidState( @@ -1266,17 +1188,17 @@ impl Runtime { ) .map_err(LoopError::InvalidState)?; let subagents = self.subagents.fresh(); - let background_jobs = BackgroundJobs { - observations: observations.clone(), - ..Default::default() - }; let builder = Agent::builder() .model(self.adapter.clone()) .telemetry(self.agentkit_telemetry()) - .add_tool_source(self.compose_with_jobs(depth, subagents, background_jobs, skills)) + .add_tool_source(self.compose_with_jobs( + depth, + subagents, + BackgroundJobs::default(), + skills, + )) .task_manager(background_task_manager()) .mutator(compactor) - .observer(observations) .transcript(transcript) .input(vec![Item::text(ItemKind::User, prompt)]); let builder = builder.cancellation(controller.handle()); @@ -1349,43 +1271,12 @@ impl Runtime { .await } - // All callers receive an already-recorded error. Do not record it again at - // a protocol boundary: that would discard the typed loop classification. pub(crate) async fn start_acp_driver_with_initial( self: &Arc, context: AcpDriverContext, claim: &mut SessionClaim, forked: Option, ) -> Result - where - I: LoopObserver + Clone + 'static, - { - self.prepare_acp_driver_with_initial(context, claim, forked) - .await - .map_err(|failure| { - AcpRuntimeError::Loop(match failure { - AcpStartupFailure::Runtime(error) => record_runtime_failure( - claim.id(), - crate::fatal::Surface::Acp, - "session_start", - error.to_string(), - ), - AcpStartupFailure::Loop(error, observations) => record_loop_failure( - claim.id(), - crate::fatal::Surface::Acp, - &error, - &observations, - ), - }) - }) - } - - async fn prepare_acp_driver_with_initial( - self: &Arc, - context: AcpDriverContext, - claim: &mut SessionClaim, - forked: Option, - ) -> Result where I: LoopObserver + Clone + 'static, { @@ -1395,8 +1286,7 @@ impl Runtime { return Err(AcpRuntimeError::Loop(format!( "this Kit runtime is fixed to {} and does not accept additional directories", self.root.display() - )) - .into()); + ))); } let request = claim.request.clone(); let session_id = request.id.clone(); @@ -1413,8 +1303,7 @@ impl Runtime { if request.resume { return Err(AcpRuntimeError::Loop( "a forked transcript requires a new session identity".into(), - ) - .into()); + )); } transcript } else if request.resume { @@ -1488,17 +1377,14 @@ impl Runtime { .task_manager(task_manager) .mutator(compactor) .observer(context.integration.as_ref().clone()) - .observer(background_jobs.observations.clone()) .transcript_observer(opened.observer) .transcript(opened.transcript) .cancellation(context.cancellation) .build() - .map_err(|error| AcpStartupFailure::Loop(error, background_jobs.observations.clone()))? + .map_err(|error| AcpRuntimeError::Loop(error.to_string()))? .start(session_config) .await - .map_err(|error| { - AcpStartupFailure::Loop(error, background_jobs.observations.clone()) - })?; + .map_err(|error| AcpRuntimeError::Loop(error.to_string()))?; let driver = AcpDriver { driver, skills: skill_catalog, @@ -1828,7 +1714,6 @@ pub(crate) struct BackgroundActivity { #[derive(Clone)] pub(crate) struct BackgroundJobs { - pub(crate) observations: crate::effects::Observations, state: Arc>, activity: watch::Sender, } @@ -1837,7 +1722,6 @@ impl Default for BackgroundJobs { fn default() -> Self { let (activity, _) = watch::channel(0); Self { - observations: crate::effects::Observations::local_session(), state: Arc::new(Mutex::new(BackgroundJobState::default())), activity, } @@ -2176,10 +2060,7 @@ impl Tool for BackgroundableCompose { crate::artifacts::directory(&self.root, &request.session_id.0, &call_id.0); let request = Self::sanitized(request)?; let _job = self.begin_background(background, &call_id, ctx); - self.background_jobs.observations.invocation_started(); - let outcome = self.inner.invoke(request, ctx).await; - self.background_jobs.observations.invocation_completed(); - match outcome { + match self.inner.invoke(request, ctx).await { Ok(mut result) => { match crate::compose_output::guard(&artifact_directory, result.result.output).await { @@ -2208,12 +2089,7 @@ impl Tool for BackgroundableCompose { Err(error) => return ToolExecutionOutcome::Failed(error), }; let _job = self.begin_background(background, &call_id, ctx); - self.background_jobs.observations.invocation_started(); - let outcome = self.inner.invoke_outcome(request, ctx).await; - if !matches!(outcome, ToolExecutionOutcome::Interrupted(_)) { - self.background_jobs.observations.invocation_completed(); - } - match outcome { + match self.inner.invoke_outcome(request, ctx).await { ToolExecutionOutcome::Completed(mut result) => { match crate::compose_output::guard(&artifact_directory, result.result.output).await { @@ -2467,28 +2343,7 @@ fn record_runtime_failure( code: &str, rendered: String, ) -> String { - record_runtime_failure_observed( - session_id, - surface, - code, - rendered, - &crate::effects::Observations::default(), - ) -} - -fn record_runtime_failure_observed( - session_id: &str, - surface: crate::fatal::Surface, - code: &str, - rendered: String, - observations: &crate::effects::Observations, -) -> String { - match crate::fatal::record_runtime_error_with_effects( - session_id, - surface, - code, - observations.snapshot(), - ) { + match crate::fatal::record_runtime_error(session_id, surface, code) { Ok(path) => format!("{rendered}; fatal log: {}", path.display()), Err(log_error) => { eprintln!("could not store fatal error log for {session_id}: {log_error}"); @@ -2501,15 +2356,9 @@ fn record_loop_failure( session_id: &str, surface: crate::fatal::Surface, error: &LoopError, - observations: &crate::effects::Observations, ) -> String { let rendered = crate::fatal::render_loop_error(error); - match crate::fatal::record_loop_error_with_effects( - session_id, - surface, - error, - observations.snapshot(), - ) { + match crate::fatal::record_loop_error(session_id, surface, error) { Ok(Some(path)) => format!("{rendered}; fatal log: {}", path.display()), Ok(None) => rendered, Err(log_error) => { diff --git a/src/runtime/tests.rs b/src/runtime/tests.rs index 4f673402..7d80cb57 100644 --- a/src/runtime/tests.rs +++ b/src/runtime/tests.rs @@ -1648,6 +1648,179 @@ async fn persistent_startup_failure_does_not_commit_new_session() { assert!(crate::session::load(root.path(), &session_id).is_err()); } +/// Run with a private HOME in a subprocess: fatal logs and durable sessions both +/// use HOME, and changing it in this process would race unrelated tests. +#[tokio::test] +async fn persistent_provider_failure_retains_private_span_context() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::{Layer, layer::SubscriberExt as _, registry::LookupSpan}; + + const CHILD: &str = "KIT_TEST_PERSISTENT_ERROR_SPANS"; + if std::env::var_os(CHILD).is_none() { + let home = tempfile::tempdir().unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = axum::Router::new().fallback(|| async { + ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(json!({"error": { + "message": "response-secret-sentinel", + "type": "invalid_request_error", + "code": 400 + }})), + ) + }); + let server = tokio::spawn( + async move { axum::serve(listener, app).await.unwrap() }.with_current_subscriber(), + ); + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "runtime::tests::persistent_provider_failure_retains_private_span_context", + "--nocapture", + ]) + .env(CHILD, "1") + .env("HOME", home.path()) + .env( + "OPENROUTER_BASE_URL", + format!("http://{address}/endpoint-secret-sentinel"), + ) + .env("NO_PROXY", "127.0.0.1") + .env_remove("OPENROUTER_MAX_COMPLETION_TOKENS") + .env_remove("OPENROUTER_TEMPERATURE") + .kill_on_drop(true); + let output = tokio::time::timeout(Duration::from_secs(30), command.output()) + .await + .expect("isolated runtime test timed out") + .unwrap(); + server.abort(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + return; + } + + struct ClosedBeforeLog { + directory: std::path::PathBuf, + names: Arc>>, + } + impl Layer for ClosedBeforeLog + where + S: tracing::Subscriber + for<'a> LookupSpan<'a>, + { + fn on_close(&self, id: tracing::span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) { + let span = ctx.span(&id).unwrap(); + let name = span.metadata().name(); + if matches!(name, "chat" | "agent.turn") && !self.directory.exists() { + self.names.lock().unwrap().push(name); + } + } + } + + let home = std::path::PathBuf::from(std::env::var_os("HOME").unwrap()); + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("AGENTS.md"), "context-secret-sentinel").unwrap(); + let mut original_error = None; + for (session_id, capture, block_log) in [ + ("span-enabled", true, false), + ("span-default-disabled", false, false), + ("span-write-failed", true, true), + ] { + let directory = home.join(".kit/errors").join(session_id); + if block_log { + // A file where the log directory belongs is a permanent write error. + std::fs::write(&directory, "not a directory").unwrap(); + } + let closed = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut settings = crate::telemetry::Settings { + capture_message_content: true, + ..Default::default() + }; + assert!(!settings.capture_error_spans); + settings.capture_error_spans = capture; + assert!(settings.endpoint.is_none()); + let subscriber = tracing_subscriber::registry() + .with( + settings + .capture_error_spans + .then_some(crate::telemetry::error_spans::ErrorSpanLayer), + ) + .with(ClosedBeforeLog { + directory: directory.clone(), + names: closed.clone(), + }); + let runtime = Runtime::with_session_provider_credentials_effort_and_openrouter_key( + root.path(), + "private/model-secret-sentinel", + crate::provider::ProviderKind::OpenRouter, + SessionRequest { + id: session_id.into(), + resume: false, + force: false, + }, + crate::credentials::CredentialStorage::Memory, + None, + Some(crate::provider::OpenRouterApiKey::new( + "api-secret-sentinel", + )), + ) + .unwrap(); + let runtime = Runtime::with_telemetry(runtime, settings).unwrap(); + let error = runtime + .run_persistent("prompt-secret-sentinel".into()) + .with_subscriber(subscriber) + .await + .unwrap_err(); + let rendered = error.split("; fatal log: ").next().unwrap(); + assert!(rendered.starts_with("provider error:"), "{error}"); + if let Some(original) = &original_error { + assert_eq!(rendered, original); + } else { + original_error = Some(rendered.to_owned()); + } + if block_log { + assert!(!error.contains("; fatal log: ")); + assert_eq!( + std::fs::read_to_string(&directory).unwrap(), + "not a directory" + ); + continue; + } + let (_, path) = error.split_once("; fatal log: ").unwrap(); + let encoded = std::fs::read_to_string(path).unwrap(); + let record: Value = serde_json::from_str(&encoded).unwrap(); + assert_eq!(record["session_id"], session_id); + assert_eq!(record["surface"], "prompt"); + assert_eq!(record["kind"], "provider"); + assert_eq!(record["code"], "provider_error"); + assert_eq!(record["message"], "provider request failed"); + assert!(!encoded.contains("secret-sentinel"), "{encoded}"); + assert!(!encoded.contains(&root.path().display().to_string())); + assert!(!encoded.contains(&home.display().to_string())); + if capture { + let fragments = record["span_context"]["fragments"].as_array().unwrap(); + assert_eq!(fragments[0]["name"], "kit.operation"); + assert_eq!(fragments[0]["fields"]["surface"], "prompt"); + for name in ["agent.turn", "chat"] { + assert!( + closed.lock().unwrap().contains(&name), + "{name} did not close before logging" + ); + assert!( + fragments.iter().any(|fragment| fragment["name"] == name), + "{encoded}" + ); + } + } else { + assert!(record.get("span_context").is_none(), "{encoded}"); + } + } +} + #[tokio::test] async fn persistent_missing_openai_credentials_do_not_commit_new_session() { let root = tempfile::tempdir().unwrap(); @@ -1722,90 +1895,3 @@ fn system_prompt_guides_compose_and_subagent_hygiene() { let max_depth_prompt = runtime.system_prompt(runtime.max_subagent_depth()); assert!(max_depth_prompt.contains("This task was delegated to you by the primary agent.")); } - -#[tokio::test] -async fn root_compose_receipts_preserve_both_entry_points_and_background_lifetime() { - for native in [false, true] { - for failing in [false, true] { - let root = tempfile::tempdir().unwrap(); - let runtime = Runtime::new(root.path(), "gpt-5.4").unwrap(); - let compose = runtime.compose(0); - let jobs = compose.backgroundable.background_jobs.clone(); - let source: Arc = Arc::new(compose.compose.clone()); - let executor: Arc = Arc::new(BasicToolExecutor::new([source])); - let session_id = SessionId::new("root-effects"); - let turn_id = TurnId::new("first"); - let permissions = Arc::new(AllowAllPermissions); - let resources: Arc = Arc::new(()); - let owned = OwnedToolContext { - session_id: session_id.clone(), - turn_id: turn_id.clone(), - metadata: MetadataMap::new(), - permissions: permissions.clone(), - resources: resources.clone(), - cancellation: None, - execution_scope: Some(ToolExecutionScope { - executor, - session_id: session_id.clone(), - turn_id: turn_id.clone(), - permissions, - resources, - cancellation: None, - }), - approved_request: None, - }; - let request = ToolRequest::new( - ToolCallId::new("effects-compose"), - ToolName::new("compose"), - json!({ - "script": if failing { "return fail(\"FAILED\", \"private failure text\")" } else { "return shell({ command: \"sleep 0.05\" })" }, "background": true, - }), - session_id, - turn_id, - ); - let invocation = async { - if native { - match compose - .backgroundable - .invoke_outcome(request, &mut owned.borrowed()) - .await - { - ToolExecutionOutcome::Completed(_) => assert!(!failing), - ToolExecutionOutcome::Failed(_) => assert!(failing), - other => panic!("unexpected native outcome: {other:?}"), - } - } else { - assert_eq!( - compose - .backgroundable - .invoke(request, &mut owned.borrowed()) - .await - .is_err(), - failing - ); - } - }; - let later_prompt = async { - tokio::time::timeout(Duration::from_secs(2), async { - while !jobs.observations.snapshot().tool_execution_start_reported { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - jobs.begin_turn(); - assert!(jobs.observations.snapshot().tool_execution_start_reported); - }; - tokio::join!(invocation, later_prompt); - let effects = jobs.observations.snapshot(); - assert_eq!( - effects.source, - crate::effects::ObservationSource::LocalSession - ); - assert!(effects.tool_execution_start_reported); - assert!(effects.tool_execution_completion_reported); - assert!(effects.observation_incomplete); - assert!(!serde_json::to_string(&effects).unwrap().contains("private")); - } - } -} diff --git a/src/telemetry.rs b/src/telemetry.rs index 8b062133..03c705f0 100644 --- a/src/telemetry.rs +++ b/src/telemetry.rs @@ -1,4 +1,6 @@ -//! Optional OpenTelemetry trace export and resolved host settings. +//! Independent opt-in local error context and OpenTelemetry trace export. + +pub(crate) mod error_spans; use agentkit_loop::{MessageCapture, TelemetryConfig}; use opentelemetry::trace::TracerProvider as _; @@ -57,6 +59,8 @@ impl FromStr for Protocol { #[derive(Clone, Debug, PartialEq, Eq)] pub struct Settings { pub endpoint: Option, + /// Collect bounded local span context in fatal error logs (independent of OTLP). + pub capture_error_spans: bool, pub protocol: Protocol, pub capture_message_content: bool, pub message_content_max_messages: usize, @@ -104,6 +108,7 @@ impl Settings { (endpoint, Protocol::Grpc) | (endpoint @ None, _) => endpoint, }; Ok(Self { + capture_error_spans: false, endpoint, protocol, capture_message_content, @@ -143,6 +148,8 @@ impl Settings { /// re-enabling export or message capture. pub fn append_cli_args(&self, command: &mut tokio::process::Command) { command + .arg("--internal-capture-error-spans") + .arg(self.capture_error_spans.to_string()) .env_remove("OTEL_EXPORTER_OTLP_ENDPOINT") .env_remove("OTEL_EXPORTER_OTLP_PROTOCOL") .env_remove("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") @@ -163,6 +170,7 @@ impl Default for Settings { fn default() -> Self { Self { endpoint: None, + capture_error_spans: false, protocol: Protocol::default(), capture_message_content: false, message_content_max_messages: DEFAULT_MESSAGE_CONTENT_MAX_MESSAGES, @@ -311,23 +319,44 @@ fn build_provider( } } -/// Installs OTLP trace export when an endpoint is configured. +/// Installs local span collection and OTLP export independently. pub fn init(settings: &Settings) -> Result, Box> { - let Some(endpoint) = settings.endpoint.as_deref() else { + if settings.endpoint.is_none() && !settings.capture_error_spans { return Ok(None); - }; - let exporter = build_exporter(endpoint, settings.protocol)?; - let provider = build_provider(exporter, settings.protocol); - let tracer = provider.tracer(env!("CARGO_PKG_NAME")); - let layer = tracing_opentelemetry::layer() - .with_tracer(tracer) - .with_location(false) - .with_threads(false) - .with_tracked_inactivity(false) - .with_target(false) - .with_filter(exported_targets()); - tracing_subscriber::registry().with(layer).try_init()?; - Ok(Some(Guard { + } + let provider = settings + .endpoint + .as_deref() + .map(|endpoint| { + build_exporter(endpoint, settings.protocol) + .map(|exporter| build_provider(exporter, settings.protocol)) + }) + .transpose()?; + let export_layer = provider.as_ref().map(|provider| { + tracing_opentelemetry::layer() + .with_tracer(provider.tracer(env!("CARGO_PKG_NAME"))) + .with_location(false) + .with_threads(false) + .with_tracked_inactivity(false) + .with_target(false) + .with_filter(exported_targets()) + }); + let result = tracing_subscriber::registry() + .with(export_layer) + .with( + settings + .capture_error_spans + .then_some(error_spans::ErrorSpanLayer), + ) + .try_init(); + // Local diagnostics are best-effort, including when a host owns the subscriber. + // Preserve the existing exporter initialization failure behavior. + if let Err(error) = result + && provider.is_some() + { + return Err(error.into()); + } + Ok(provider.map(|provider| Guard { provider: Some(provider), protocol: settings.protocol, })) @@ -538,6 +567,75 @@ mod tests { assert!(invalid.agentkit_config().is_err()); } + #[test] + fn child_args_propagate_both_local_capture_values() { + for enabled in [false, true] { + let settings = Settings { + capture_error_spans: enabled, + ..Settings::default() + }; + let mut command = tokio::process::Command::new("kit"); + settings.append_cli_args(&mut command); + let args: Vec<_> = command + .as_std() + .get_args() + .map(|value| value.to_string_lossy().into_owned()) + .collect(); + assert_eq!( + &args[..2], + &[ + "--internal-capture-error-spans".to_owned(), + enabled.to_string() + ] + ); + } + } + + #[test] + fn error_span_initialization_matrix() { + // init owns a global subscriber, so exercise each independent combination + // in a fresh test process rather than leaking state to parallel tests. + const CHILD: &str = "KIT_TEST_ERROR_SPAN_INIT"; + let Ok(mode) = std::env::var(CHILD) else { + for mode in ["00", "01", "10", "11", "occupied"] { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "telemetry::tests::error_span_initialization_matrix", + "--nocapture", + ]) + .env(CHILD, mode) + .output() + .unwrap(); + assert!( + output.status.success(), + "{mode}: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + return; + }; + let runtime = tokio::runtime::Runtime::new().unwrap(); + let _runtime = runtime.enter(); + if mode == "occupied" { + tracing::subscriber::set_global_default(tracing_subscriber::registry()).unwrap(); + } + let settings = Settings { + capture_error_spans: mode.starts_with('1') || mode == "occupied", + endpoint: mode.ends_with('1').then(|| "http://127.0.0.1:1".into()), + ..Settings::default() + }; + let guard = super::init(&settings).unwrap(); + assert_eq!(guard.is_some(), settings.endpoint.is_some()); + let operation = super::error_spans::operation("prompt"); + operation.in_scope(|| { let _child = tracing::info_span!(target: "agentkit_loop", "chat", "gen_ai.operation.name" = "chat"); }); + assert_eq!( + super::error_spans::snapshot(&operation).is_some(), + settings.capture_error_spans && mode != "occupied" + ); + } + #[test] fn child_args_propagate_protocol_endpoint_explicit_false_and_bounds() { let settings = Settings::try_new_with_protocol( @@ -581,6 +679,8 @@ mod tests { assert_eq!( args, [ + "--internal-capture-error-spans", + "false", "--otel-endpoint", "http://collector:4318/v1/traces", "--otel-protocol", @@ -607,7 +707,7 @@ mod tests { .collect(); assert_eq!( - &args[..4], + &args[2..6], ["--otel-endpoint", "", "--otel-protocol", "grpc"] ); assert!( diff --git a/src/telemetry/error_spans.rs b/src/telemetry/error_spans.rs new file mode 100644 index 00000000..500c42a5 --- /dev/null +++ b/src/telemetry/error_spans.rs @@ -0,0 +1,426 @@ +//! Opt-in, operation-local tracing history for existing fatal diagnostics. +//! This is a partial history, not an effects ledger or a causal error chain. + +use std::{ + collections::BTreeMap, + fmt, + sync::{Arc, Mutex}, +}; + +use serde::{Deserialize, Serialize}; +use tracing::{ + Span, Subscriber, + field::{Field, Visit}, + span::{Attributes, Id, Record}, +}; +use tracing_subscriber::{Layer, Registry, layer::Context, registry::LookupSpan}; + +const MAX_FRAGMENTS: usize = 24; +const MAX_DEPTH: usize = 8; +const MAX_FIELDS: usize = 6; +const MAX_VALUE_BYTES: usize = 32; +const MAX_SNAPSHOT_BYTES: usize = 12 * 1024; +const TARGET: &str = "kit::telemetry::error_spans"; + +#[cfg(test)] +mod task_manager_tests; + +/// The future instrumented with this span must include execution AND error logging. +/// Each call starts a fresh history even when nested within another operation. +pub(crate) fn operation(surface: &'static str) -> Span { + tracing::info_span!(target: TARGET, parent: Span::current(), "kit.operation", surface) +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Snapshot { + fragments: Vec, + /// Indicates a collection bound, not whether observations are complete. + truncated: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Fragment { + name: String, + parent: Option, + fields: BTreeMap, +} + +impl Snapshot { + pub(crate) fn valid(&self) -> bool { + !self.fragments.is_empty() + && self.fragments.len() <= MAX_FRAGMENTS + && self.fragments.iter().enumerate().all(|(index, fragment)| { + (if index == 0 { + fragment.name == "kit.operation" && fragment.parent.is_none() + } else { + approved_name(&fragment.name) + && fragment.parent.is_some_and(|parent| parent < index) + }) && { + let mut parent = fragment.parent; + let mut depth = 0; + while let Some(index) = parent { + depth += 1; + if depth > MAX_DEPTH { + return false; + } + parent = self.fragments[index].parent; + } + true + } && fragment.fields.len() <= MAX_FIELDS + && fragment + .fields + .iter() + .all(|(key, value)| approved_value(key, value)) + }) + && serde_json::to_vec_pretty(self).is_ok_and(|bytes| bytes.len() <= MAX_SNAPSHOT_BYTES) + } +} + +#[derive(Clone)] +struct Capture { + history: Arc>, + index: usize, + depth: usize, +} + +pub(crate) struct ErrorSpanLayer; + +impl Layer for ErrorSpanLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) { + let Some(span) = ctx.span(id) else { return }; + let metadata = attrs.metadata(); + let root = metadata.target() == TARGET && metadata.name() == "kit.operation"; + let capture = if root { + Capture { + history: Arc::new(Mutex::new(Snapshot { + fragments: Vec::new(), + truncated: false, + })), + index: 0, + depth: 0, + } + } else { + let parent = if attrs.is_contextual() { + ctx.lookup_current() + } else { + attrs.parent().and_then(|parent| ctx.span(parent)) + }; + let Some(mut capture) = + parent.and_then(|parent| parent.extensions().get::().cloned()) + else { + return; + }; + capture.depth = capture.depth.saturating_add(1); + if capture.depth > MAX_DEPTH { + if let Ok(mut history) = capture.history.try_lock() { + history.truncated = true; + } + return; + } + capture + }; + let approved = + root || (metadata.target() == "agentkit_loop" && approved_name(metadata.name())); + let mut capture = capture; + if approved { + // No extension lock is held while locking the operation store. + let Ok(mut history) = capture.history.try_lock() else { + return; + }; + if history.fragments.len() < MAX_FRAGMENTS { + let mut visitor = Fields::default(); + attrs.record(&mut visitor); + let parent = (!root).then_some(capture.index); + capture.index = history.fragments.len(); + history.fragments.push(Fragment { + name: metadata.name().into(), + parent, + fields: visitor.values, + }); + history.truncated |= visitor.truncated; + } else { + history.truncated = true; + // Do not let records on an omitted span update its parent's fields. + return; + } + } + span.extensions_mut().insert(capture); + } + + fn on_record(&self, id: &Id, values: &Record<'_>, ctx: Context<'_, S>) { + let Some(span) = ctx.span(id) else { return }; + let metadata = span.metadata(); + if !(metadata.target() == TARGET && metadata.name() == "kit.operation" + || metadata.target() == "agentkit_loop" && approved_name(metadata.name())) + { + return; + } + let capture = span.extensions().get::().cloned(); + let Some(capture) = capture else { return }; + let mut visitor = Fields::default(); + values.record(&mut visitor); + let Ok(mut history) = capture.history.try_lock() else { + return; + }; + let mut truncated = visitor.truncated; + if let Some(fragment) = history.fragments.get_mut(capture.index) { + for (key, value) in visitor.values { + if fragment.fields.contains_key(&key) || fragment.fields.len() < MAX_FIELDS { + fragment.fields.insert(key, value); + } else { + truncated = true; + } + } + } + history.truncated |= truncated; + } +} + +/// Uses the retained operation's extensions, including children already closed. +/// Without the layer there are no buffers and no traversal/serialization. +pub(crate) fn snapshot(span: &Span) -> Option { + span.with_subscriber(|(id, dispatch)| { + dispatch.downcast_ref::()?; + let registry = dispatch.downcast_ref::()?; + let span = registry.span(id)?; + let capture = span.extensions().get::().cloned()?; + let mut snapshot = capture.history.try_lock().ok()?.clone(); + // Keep encoding outside all locks. A failed/contended capture is optional. + while serde_json::to_vec_pretty(&snapshot).ok()?.len() > MAX_SNAPSHOT_BYTES { + snapshot.fragments.pop()?; + snapshot.truncated = true; + } + Some(snapshot) + }) + .flatten() +} + +fn approved_name(name: &str) -> bool { + matches!(name, "agent.turn" | "agent.execute_tool" | "chat") +} + +fn approved_value(key: &str, value: &serde_json::Value) -> bool { + match value { + serde_json::Value::String(value) if value.len() <= MAX_VALUE_BYTES => match key { + "surface" => matches!(value.as_str(), "prompt" | "a2a" | "acp" | "acp_autonomous"), + "gen_ai.operation.name" => { + matches!(value.as_str(), "invoke_agent" | "execute_tool" | "chat") + } + "launch_kind" => matches!(value.as_str(), "plain" | "approved"), + "error.type" => matches!(value.as_str(), "tool_error" | "provider_error"), + _ => false, + }, + serde_json::Value::Number(value) => { + matches!( + key, + "transcript.len" | "gen_ai.usage.input_tokens" | "gen_ai.usage.output_tokens" + ) && value.as_u64().is_some_and(|value| value <= u32::MAX.into()) + } + serde_json::Value::Bool(_) => key == "saw_tool_call", + _ => false, + } +} + +#[derive(Default)] +struct Fields { + values: BTreeMap, + truncated: bool, +} + +impl Fields { + fn insert(&mut self, field: &Field, value: serde_json::Value) { + if !approved_value(field.name(), &value) { + return; + } + if self.values.contains_key(field.name()) || self.values.len() < MAX_FIELDS { + self.values.insert(field.name().into(), value); + } else { + self.truncated = true; + } + } +} + +impl Visit for Fields { + fn record_str(&mut self, field: &Field, value: &str) { + // Reject before allocation; identifiers/content are intentionally not collected. + if value.len() <= MAX_VALUE_BYTES { + self.insert(field, value.into()); + } + } + fn record_u64(&mut self, field: &Field, value: u64) { + self.insert(field, value.into()); + } + fn record_i64(&mut self, field: &Field, value: i64) { + self.insert(field, value.into()); + } + fn record_bool(&mut self, field: &Field, value: bool) { + self.insert(field, value.into()); + } + fn record_debug(&mut self, _: &Field, _: &dyn fmt::Debug) { + // Includes Display wrappers: never format arbitrary user/provider payloads. + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tracing::Instrument as _; + use tracing_subscriber::prelude::*; + + #[test] + fn disabled_layer_has_no_capture() { + tracing::subscriber::with_default(tracing_subscriber::registry(), || { + let operation = operation("prompt"); + assert!(snapshot(&operation).is_none()); + operation.with_subscriber(|(id, dispatch)| { + let registry = dispatch.downcast_ref::().unwrap(); + assert!( + registry + .span(id) + .unwrap() + .extensions() + .get::() + .is_none() + ); + }); + }); + } + + #[test] + fn closed_children_and_late_records_survive_without_exporter() { + tracing::subscriber::with_default( + tracing_subscriber::registry().with(ErrorSpanLayer), + || { + let operation = operation("prompt"); + operation.in_scope(|| { + let child = tracing::info_span!(target: "agentkit_loop", "agent.execute_tool", + launch_kind = "plain", "error.type" = tracing::field::Empty); + child.record("error.type", "tool_error"); + }); + let context = snapshot(&operation).unwrap(); + assert!(context.valid()); + assert_eq!(context.fragments.len(), 2); + assert_eq!(context.fragments[1].parent, Some(0)); + assert_eq!(context.fragments[1].fields["error.type"], "tool_error"); + assert_eq!(context.fragments[1].fields["launch_kind"], "plain"); + }, + ); + } + + #[test] + fn privacy_rejects_content_identifiers_debug_and_untrusted_targets() { + struct NeverFormat; + impl fmt::Debug for NeverFormat { + fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result { + panic!("must not format"); + } + } + tracing::subscriber::with_default( + tracing_subscriber::registry().with(ErrorSpanLayer), + || { + let operation = operation("prompt"); + operation.in_scope(|| { + let child = tracing::info_span!(target: "agentkit_loop", "chat", + "gen_ai.input.messages" = ?NeverFormat, + "gen_ai.output.messages" = "SECRET", + "gen_ai.conversation.id" = ?NeverFormat, + "gen_ai.operation.name" = "chat", + "gen_ai.usage.input_tokens" = u64::MAX, + "error.type" = ?NeverFormat); + child.record("gen_ai.output.messages", "SECRET".repeat(100_000)); + child.record("gen_ai.operation.name", "https://secret.invalid/token"); + child.record("gen_ai.conversation.id", "../../SECRET"); + let _untrusted = tracing::info_span!(target: "untrusted", "chat", "gen_ai.operation.name" = "chat"); + }); + let context = snapshot(&operation).unwrap(); + assert_eq!(context.fragments.len(), 2); + assert_eq!(context.fragments[1].fields.len(), 1); + assert_eq!(context.fragments[1].fields["gen_ai.operation.name"], "chat"); + let encoded = serde_json::to_string(&context).unwrap(); + assert!(!encoded.contains("SECRET")); + assert!(!encoded.contains("secret.invalid")); + }, + ); + } + + #[test] + fn bounds_depth_count_and_snapshot_size() { + tracing::subscriber::with_default( + tracing_subscriber::registry().with(ErrorSpanLayer), + || { + let operation = operation("prompt"); + operation.in_scope(|| { + for _ in 0..1000 { + let _child = tracing::info_span!(target: "agentkit_loop", "chat", "gen_ai.operation.name" = "chat"); + } + }); + let context = snapshot(&operation).unwrap(); + assert_eq!(context.fragments.len(), MAX_FRAGMENTS); + assert!(context.truncated); + assert!(context.valid()); + + let deep = super::operation("acp"); + let mut parent = deep.clone(); + for _ in 0..100 { + parent = tracing::info_span!(target: "agentkit_loop", parent: &parent, "chat"); + } + let context = snapshot(&deep).unwrap(); + assert!(context.truncated); + assert_eq!(context.fragments.len(), MAX_DEPTH + 1); + }, + ); + } + + #[test] + fn unavailable_capture_is_omitted() { + tracing::subscriber::with_default( + tracing_subscriber::registry().with(ErrorSpanLayer), + || { + let operation = operation("prompt"); + operation.with_subscriber(|(id, dispatch)| { + let registry = dispatch.downcast_ref::().unwrap(); + let capture = registry + .span(id) + .unwrap() + .extensions() + .get::() + .unwrap() + .clone(); + let _lock = capture.history.lock().unwrap(); + assert!(snapshot(&operation).is_none()); + operation.record("surface", "a2a"); // contended collection cannot block or fail + }); + assert!(snapshot(&operation).is_some()); + }, + ); + } + + #[tokio::test] + async fn explicitly_instrumented_spawns_keep_separate_operation_histories() { + use tracing::instrument::WithSubscriber as _; + let subscriber = tracing_subscriber::registry().with(ErrorSpanLayer); + async { + let first = operation("prompt"); + let second = operation("a2a"); + let one = tokio::spawn(async { + tokio::task::yield_now().await; + let _child = tracing::info_span!(target: "agentkit_loop", "chat", "gen_ai.operation.name" = "chat"); + }.instrument(first.clone()).with_current_subscriber()); + let two = tokio::spawn(async { + let _child = tracing::info_span!(target: "agentkit_loop", "agent.execute_tool", launch_kind = "approved"); + tokio::task::yield_now().await; + }.instrument(second.clone()).with_current_subscriber()); + one.await.unwrap(); + two.await.unwrap(); + assert_eq!(snapshot(&first).unwrap().fragments[1].name, "chat"); + assert_eq!(snapshot(&second).unwrap().fragments[1].name, "agent.execute_tool"); + assert_eq!(snapshot(&first).unwrap().fragments.len(), 2); + assert_eq!(snapshot(&second).unwrap().fragments.len(), 2); + }.with_subscriber(subscriber).await; + } +} diff --git a/src/telemetry/error_spans/task_manager_tests.rs b/src/telemetry/error_spans/task_manager_tests.rs new file mode 100644 index 00000000..181bbd83 --- /dev/null +++ b/src/telemetry/error_spans/task_manager_tests.rs @@ -0,0 +1,207 @@ +//! Real async-manager execution documents the upstream spawn boundary without +//! wrapping or patching it. Set KIT_TEST_REQUIRE_TASK_MANAGER_ANCESTRY=1 to turn +//! the known limitation assertion into the desired (currently failing) contract. + +use std::{ + collections::VecDeque, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use agentkit_core::{ + FinishReason, Item, ItemKind, MetadataMap, Part, SessionId, ToolCallPart, ToolOutput, + ToolResultPart, TurnCancellation, +}; +use agentkit_loop::{ + Agent, LoopError, ModelAdapter, ModelSession, ModelTurn, ModelTurnEvent, ModelTurnResult, + SessionConfig, TurnRequest, +}; +use agentkit_task_manager::AsyncTaskManager; +use agentkit_tools_core::{ + Tool, ToolContext, ToolError, ToolName, ToolRegistry, ToolRequest, ToolResult, ToolSpec, +}; +use async_trait::async_trait; +use serde_json::json; +use tracing::Instrument; +use tracing_subscriber::prelude::*; + +use super::{ErrorSpanLayer, operation, snapshot}; + +struct FixtureAdapter; +struct FixtureSession(bool); +struct FixtureTurn(VecDeque); + +#[async_trait] +impl ModelAdapter for FixtureAdapter { + type Session = FixtureSession; + + async fn start_session(&self, _: SessionConfig) -> Result { + Ok(FixtureSession(false)) + } +} + +#[async_trait] +impl ModelSession for FixtureSession { + type Turn = FixtureTurn; + + async fn begin_turn( + &mut self, + _: TurnRequest, + _: Option, + ) -> Result { + if self.0 { + return Err(LoopError::Provider("fixture final failure".into())); + } + self.0 = true; + let call = ToolCallPart::new("probe-call", "probe", json!({})); + Ok(FixtureTurn(VecDeque::from([ + ModelTurnEvent::ToolCall(call.clone()), + ModelTurnEvent::Finished(ModelTurnResult { + finish_reason: FinishReason::ToolCall, + output_items: vec![Item::new(ItemKind::Assistant, vec![Part::ToolCall(call)])], + usage: None, + metadata: MetadataMap::new(), + model: None, + response_id: None, + }), + ]))) + } +} + +#[async_trait] +impl ModelTurn for FixtureTurn { + async fn next_event( + &mut self, + _: Option, + ) -> Result, LoopError> { + Ok(self.0.pop_front()) + } +} + +struct Probe { + spec: ToolSpec, + executed: Arc, +} + +#[async_trait] +impl Tool for Probe { + fn spec(&self) -> &ToolSpec { + &self.spec + } + + async fn invoke( + &self, + request: ToolRequest, + _: &mut ToolContext<'_>, + ) -> Result { + // Same allowlisted shape as the control span, but a distinct numeric + // marker so the loop's own inference spans cannot satisfy the assertion. + let span = tracing::info_span!(target: "agentkit_loop", "chat", + gen_ai.operation.name = "chat", gen_ai.usage.output_tokens = 4242_u64); + assert!( + !span.is_disabled(), + "global subscriber must reach the spawned task" + ); + async { + self.executed.store(true, Ordering::SeqCst); + Ok(ToolResult::new(ToolResultPart::success( + request.call_id, + ToolOutput::text("probe completed"), + ))) + } + .instrument(span) + .await + } +} + +#[tokio::test] +async fn async_manager_does_not_retain_invocation_ancestry() { + const TEST: &str = "telemetry::error_spans::task_manager_tests::async_manager_does_not_retain_invocation_ancestry"; + const CHILD: &str = "KIT_TASK_MANAGER_ANCESTRY_TEST_CHILD"; + if std::env::var(CHILD).as_deref() != Ok(TEST) { + // A global subscriber in a fresh process tests span propagation, not + // the separate failure to propagate a thread-local default subscriber. + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST, "--nocapture"]) + .env(CHILD, TEST) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("running 1 test"), + "isolated child must run the exact ancestry test" + ); + return; + } + tracing_subscriber::registry().with(ErrorSpanLayer).init(); + let executed = Arc::new(AtomicBool::new(false)); + let tools = ToolRegistry::new().with(Probe { + spec: ToolSpec::new(ToolName::new("probe"), "probe", json!({"type": "object"})), + executed: executed.clone(), + }); + let agent = Agent::builder() + .model(FixtureAdapter) + .add_tool_source(tools) + // Default routing is foreground: detachment is not needed to lose ancestry. + .task_manager(AsyncTaskManager::new()) + .input(vec![Item::text(ItemKind::User, "probe")]) + .build() + .unwrap(); + let root = operation("prompt"); + let error = async { + tracing::info_span!(target: "agentkit_loop", "chat", + gen_ai.operation.name = "chat", gen_ai.usage.output_tokens = 4241_u64) + .in_scope(|| {}); + let mut driver = agent + .start(SessionConfig::new(SessionId::new("ancestry-test")).without_cache()) + .await + .unwrap(); + for _ in 0..8 { + if let Err(error) = driver.next().await { + return error; + } + } + panic!("fixture did not reach the final provider failure"); + } + .instrument(root.clone()) + .await; + assert!(matches!(error, LoopError::Provider(message) if message == "fixture final failure")); + assert!(executed.load(Ordering::SeqCst)); + let snapshot = snapshot(&root).expect("operation retains history at the actual error boundary"); + assert!(snapshot.valid()); + assert!(!snapshot.truncated); + assert!( + snapshot + .fragments + .iter() + .any(|fragment| fragment.name == "agent.execute_tool") + ); + let has_marker = |marker| { + snapshot.fragments.iter().any(|fragment| { + fragment.name == "chat" + && fragment.fields.get("gen_ai.usage.output_tokens") == Some(&json!(marker)) + }) + }; + assert!( + has_marker(4241), + "allowlisted control span must be captured" + ); + if std::env::var("KIT_TEST_REQUIRE_TASK_MANAGER_ANCESTRY").as_deref() == Ok("1") { + assert!( + has_marker(4242), + "actual tool invocation lost its dispatch/operation ancestry" + ); + } else { + assert!( + !has_marker(4242), + "upstream propagation changed; revisit the documented scope" + ); + } +} diff --git a/src/tools/observed.rs b/src/tools/observed.rs index 6703577d..f67674e0 100644 --- a/src/tools/observed.rs +++ b/src/tools/observed.rs @@ -1,5 +1,9 @@ -//! Transparent lifecycle reporting for the hidden tools behind `compose`. -//! Effects observations are independent of the opt-in stderr display channel. +//! Lifecycle reporting for the hidden tools behind `compose`. +//! +//! The wrapper is transparent to the model and to compose: it forwards the +//! spec, permission requests, and invocation untouched, and only publishes +//! start/finish events on the runtime side channel (see [`crate::events`]) so +//! a client can draw what a Runlet program is doing while it runs. use std::{sync::Arc, time::Instant}; @@ -9,52 +13,23 @@ use agentkit_tools_core::{ ToolSpec, }; use async_trait::async_trait; + use serde_json::{Value, json}; -use crate::{ - effects::Observations, - events::{self, RuntimeEvent, summarize_input, summarize_output}, -}; +use crate::events::{self, RuntimeEvent, summarize_input, summarize_output}; /// Wraps a tool so its calls appear on the runtime side channel. -pub struct Observed { - tool: T, - observations: Option, -} +pub struct Observed(T); impl Observed { pub const fn new(tool: T) -> Self { - Self { - tool, - observations: None, - } - } - - pub(crate) fn with_observations(mut self, observations: Observations) -> Self { - self.observations = Some(observations); - self - } - - fn start(&self, request: &ToolRequest) -> Option { - if let Some(observations) = &self.observations { - observations.invocation_started(); - } - DisplayInvocation::start(request) - } - - fn finish(&self, display: Option, result: Result<&ToolResult, &ToolError>) { - if let Some(observations) = &self.observations { - observations.invocation_completed(); - } - if let Some(display) = display { - display.finish(result); - } + Self(tool) } } -/// Wraps a dynamically dispatched tool without hiding specs or native outcomes. -pub(crate) fn shared(tool: Arc, observations: Observations) -> impl Tool { - Observed::new(SharedTool(tool)).with_observations(observations) +/// Wraps a dynamically dispatched tool without hiding its changing spec. +pub(crate) fn shared(tool: Arc) -> impl Tool { + Observed(SharedTool(tool)) } struct SharedTool(Arc); @@ -64,15 +39,18 @@ impl Tool for SharedTool { fn spec(&self) -> &ToolSpec { self.0.spec() } + fn current_spec(&self) -> Option { self.0.current_spec() } + fn proposed_requests( &self, request: &ToolRequest, ) -> Result>, ToolError> { self.0.proposed_requests(request) } + async fn invoke( &self, request: ToolRequest, @@ -80,6 +58,7 @@ impl Tool for SharedTool { ) -> Result { self.0.invoke(request, context).await } + async fn invoke_outcome( &self, request: ToolRequest, @@ -92,42 +71,48 @@ impl Tool for SharedTool { #[async_trait] impl Tool for Observed { fn spec(&self) -> &ToolSpec { - self.tool.spec() + self.0.spec() } + fn current_spec(&self) -> Option { - self.tool.current_spec() + self.0.current_spec() } + fn proposed_requests( &self, request: &ToolRequest, ) -> Result>, ToolError> { - self.tool.proposed_requests(request) + self.0.proposed_requests(request) } + async fn invoke( &self, request: ToolRequest, context: &mut ToolContext<'_>, ) -> Result { - let display = self.start(&request); - let outcome = self.tool.invoke(request, context).await; - self.finish(display, outcome.as_ref()); + let display = DisplayInvocation::start(&request); + let outcome = self.0.invoke(request, context).await; + if let Some(display) = display { + display.finish(outcome.as_ref()); + } outcome } + async fn invoke_outcome( &self, request: ToolRequest, context: &mut ToolContext<'_>, ) -> ToolExecutionOutcome { - let display = self.start(&request); - let outcome = self.tool.invoke_outcome(request, context).await; - match &outcome { - ToolExecutionOutcome::Completed(result) => self.finish(display, Ok(result)), - ToolExecutionOutcome::Failed(error) - | ToolExecutionOutcome::FailedBeforeInvocation(error) => { - self.finish(display, Err(error)) + let display = DisplayInvocation::start(&request); + let outcome = self.0.invoke_outcome(request, context).await; + if let Some(display) = display { + match &outcome { + ToolExecutionOutcome::Completed(result) => display.finish(Ok(result)), + ToolExecutionOutcome::Failed(error) + | ToolExecutionOutcome::FailedBeforeInvocation(error) => display.finish(Err(error)), + // An approval interruption is not a completed invocation. + ToolExecutionOutcome::Interrupted(_) => {} } - // Neither interruption nor dropping an in-flight future is completion. - ToolExecutionOutcome::Interrupted(_) => {} } outcome } @@ -138,6 +123,7 @@ struct DisplayInvocation { tool: String, started: Instant, } + impl DisplayInvocation { fn start(request: &ToolRequest) -> Option { if !events::enabled() { @@ -157,6 +143,7 @@ impl DisplayInvocation { started: Instant::now(), }) } + fn finish(self, result: Result<&ToolResult, &ToolError>) { let (ok, summary) = match result { Ok(result) => ( @@ -193,161 +180,119 @@ mod tests { ToolName, }; - #[derive(Clone, Copy)] + #[derive(Clone, Copy, Debug)] enum Mode { - Complete, - Pending, + Completed, Failed, + FailedBeforeInvocation, Cancelled, Interrupted, } - struct Fixture { + + struct NativeTool { spec: ToolSpec, mode: Mode, } - impl Fixture { - fn new(mode: Mode) -> Self { - Self { - spec: ToolSpec::new(ToolName::new("fixture"), "fixture", json!({})), - mode, - } - } - } + #[async_trait] - impl Tool for Fixture { + impl Tool for NativeTool { fn spec(&self) -> &ToolSpec { &self.spec } + async fn invoke( &self, - request: ToolRequest, + _: ToolRequest, _: &mut ToolContext<'_>, ) -> Result { - match self.mode { - Mode::Pending => std::future::pending().await, - Mode::Complete => Ok(ToolResult::new(ToolResultPart::success( - request.call_id, - ToolOutput::text("private result"), - ))), - Mode::Failed => Err(ToolError::ExecutionFailed("failed".into())), - Mode::Cancelled => Err(ToolError::Cancelled), - Mode::Interrupted => panic!("native outcome must not use invoke fallback"), - } + panic!("wrapper must forward invoke_outcome, not use the invoke fallback") } + async fn invoke_outcome( &self, request: ToolRequest, - context: &mut ToolContext<'_>, + _: &mut ToolContext<'_>, ) -> ToolExecutionOutcome { - if matches!(self.mode, Mode::Interrupted) { - return ToolExecutionOutcome::Interrupted(ToolInterruption::ApprovalRequired( - ApprovalRequest::new( + match self.mode { + Mode::Completed => ToolExecutionOutcome::Completed(ToolResult::new( + ToolResultPart::success(request.call_id, ToolOutput::text("done")), + )), + Mode::Failed => { + ToolExecutionOutcome::Failed(ToolError::ExecutionFailed("failed".into())) + } + Mode::FailedBeforeInvocation => ToolExecutionOutcome::FailedBeforeInvocation( + ToolError::Unavailable("not started".into()), + ), + Mode::Cancelled => ToolExecutionOutcome::Failed(ToolError::Cancelled), + Mode::Interrupted => ToolExecutionOutcome::Interrupted( + ToolInterruption::ApprovalRequired(ApprovalRequest::new( "approval", - "fixture", + "native", ApprovalReason::PolicyRequiresConfirmation, "approval", - ), - )); + )), + ), } - match self.invoke(request, context).await { - Ok(result) => ToolExecutionOutcome::Completed(result), - Err(error) => ToolExecutionOutcome::Failed(error), - } - } - } - fn context() -> OwnedToolContext { - OwnedToolContext { - session_id: SessionId::new("session"), - turn_id: TurnId::new("turn"), - metadata: MetadataMap::new(), - permissions: Arc::new(AllowAllPermissions), - resources: Arc::new(()), - cancellation: None, - execution_scope: None, - approved_request: None, - } - } - fn request() -> ToolRequest { - ToolRequest::new( - ToolCallId::new("private-call"), - ToolName::new("fixture"), - json!({"secret": "private arguments"}), - SessionId::new("session"), - TurnId::new("turn"), - ) - } - - #[tokio::test] - async fn observations_do_not_depend_on_display_events() { - if crate::effects::isolated_test( - "tools::observed::tests::observations_do_not_depend_on_display_events", - ) { - return; } - assert!(!events::enabled()); - let observations = Observations::local_session(); - let tool = - Observed::new(Fixture::new(Mode::Complete)).with_observations(observations.clone()); - tool.invoke(request(), &mut context().borrowed()) - .await - .unwrap(); - assert!(observations.snapshot().tool_execution_start_reported); - assert!(observations.snapshot().tool_execution_completion_reported); - assert!( - !serde_json::to_string(&observations.snapshot()) - .unwrap() - .contains("private") - ); - } - - #[tokio::test] - async fn dropping_running_invocation_does_not_report_completion() { - let observations = Observations::local_session(); - let tool = - Observed::new(Fixture::new(Mode::Pending)).with_observations(observations.clone()); - let owned = context(); - let mut context = owned.borrowed(); - let mut invocation = Box::pin(tool.invoke(request(), &mut context)); - tokio::select! { biased; _ = &mut invocation => panic!("pending"), () = tokio::task::yield_now() => {} } - drop(invocation); - assert!(observations.snapshot().tool_execution_start_reported); - assert!(!observations.snapshot().tool_execution_completion_reported); - assert!(observations.snapshot().observation_incomplete); } #[tokio::test] - async fn shared_wrapper_preserves_native_interruption_failure_and_cancellation() { + async fn both_wrappers_preserve_native_outcomes() { for mode in [ - Mode::Complete, + Mode::Completed, Mode::Failed, + Mode::FailedBeforeInvocation, Mode::Cancelled, Mode::Interrupted, ] { - let observations = Observations::local_session(); - let tool = shared(Arc::new(Fixture::new(mode)), observations.clone()); - let outcome = tool - .invoke_outcome(request(), &mut context().borrowed()) - .await; - match mode { - Mode::Complete => assert!(matches!(outcome, ToolExecutionOutcome::Completed(_))), - Mode::Failed => assert!(matches!( - outcome, - ToolExecutionOutcome::Failed(ToolError::ExecutionFailed(_)) - )), - Mode::Cancelled => assert!(matches!( - outcome, - ToolExecutionOutcome::Failed(ToolError::Cancelled) - )), - Mode::Interrupted => { - assert!(matches!(outcome, ToolExecutionOutcome::Interrupted(_))) - } - Mode::Pending => unreachable!(), + for dynamic in [false, true] { + let native = NativeTool { + spec: ToolSpec::new(ToolName::new("native"), "native", json!({})), + mode, + }; + let tool: Box = if dynamic { + Box::new(shared(Arc::new(native))) + } else { + Box::new(Observed::new(native)) + }; + let context = OwnedToolContext { + session_id: SessionId::new("session"), + turn_id: TurnId::new("turn"), + metadata: MetadataMap::new(), + permissions: Arc::new(AllowAllPermissions), + resources: Arc::new(()), + cancellation: None, + execution_scope: None, + approved_request: None, + }; + let request = ToolRequest::new( + ToolCallId::new("call"), + ToolName::new("native"), + json!({}), + context.session_id.clone(), + context.turn_id.clone(), + ); + let outcome = tool.invoke_outcome(request, &mut context.borrowed()).await; + let preserved = match (mode, outcome) { + (Mode::Completed, ToolExecutionOutcome::Completed(result)) => { + result.result.output == ToolOutput::text("done") + } + ( + Mode::Failed, + ToolExecutionOutcome::Failed(ToolError::ExecutionFailed(message)), + ) => message == "failed", + ( + Mode::FailedBeforeInvocation, + ToolExecutionOutcome::FailedBeforeInvocation(ToolError::Unavailable( + message, + )), + ) => message == "not started", + (Mode::Cancelled, ToolExecutionOutcome::Failed(ToolError::Cancelled)) => true, + (Mode::Interrupted, ToolExecutionOutcome::Interrupted(_)) => true, + _ => false, + }; + assert!(preserved, "{mode:?}, shared={dynamic}"); } - assert!(observations.snapshot().tool_execution_start_reported); - assert_eq!( - observations.snapshot().tool_execution_completion_reported, - !matches!(mode, Mode::Interrupted) - ); } } } diff --git a/src/tools/subagent.rs b/src/tools/subagent.rs index 99103675..8e989e0a 100644 --- a/src/tools/subagent.rs +++ b/src/tools/subagent.rs @@ -12,6 +12,7 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use tokio::sync::{Mutex as AsyncMutex, OwnedSemaphorePermit, Semaphore, oneshot}; +use tracing::Instrument as _; const MAX_LIVE_SUBAGENTS: usize = 120; const MAX_DISPLAY_NAME_LEN: usize = 32; @@ -68,8 +69,7 @@ fn reserve_name(used: &mut HashSet, name: &str) -> bool { } fn child_error_is_terminal(error: &ChildError, child: &ChildSession) -> bool { - match error.root() { - ChildError::Observed { .. } => unreachable!("root unwraps observations"), + match error { ChildError::TerminalCancelled | ChildError::TerminalFailed(_) => true, ChildError::Cancelled | ChildError::Failed(_) => child.is_closed(), } @@ -237,16 +237,8 @@ struct CreateOptions { cwd: Option, } -struct ForkRequest { - prior: SubagentValue, - prompt: String, - name: Option, - contract: Option>, -} - -struct ForkReply { - effects: crate::effects::PossibleEffects, - value: Result, +struct ForkSuccess { + value: SubagentValue, acknowledge: oneshot::Sender<()>, } @@ -449,11 +441,9 @@ impl Subagents { return Err(error); } }; - let effects = output.possible_effects(); let (output, updates) = turn_output(output, contract); let mut locked = state.lock().await; - self.check_active(&locked) - .map_err(|error| error.with_effects(effects))?; + self.check_active(&locked)?; locked.status = SubagentStatus::Idle; locked.outcome = Some(GenerationOutcome::Success); locked.generation_finished_at_unix_ms = Some(events::now_millis()); @@ -514,11 +504,9 @@ impl Subagents { .await { Ok(output) => { - let effects = output.possible_effects(); let (output, updates) = turn_output(output, contract); let mut locked = state.lock().await; - self.check_active(&locked) - .map_err(|error| error.with_effects(effects))?; + self.check_active(&locked)?; locked.status = SubagentStatus::Idle; locked.handle_generation = generation; locked.outcome = Some(GenerationOutcome::Success); @@ -560,17 +548,13 @@ impl Subagents { async fn fork( &self, - parent_session_id: String, - request: ForkRequest, + prior: SubagentValue, + prompt: String, + name: Option, depth: usize, cancellation: TurnCancellation, + contract: Option>, ) -> Result { - let ForkRequest { - prior, - prompt, - name, - contract, - } = request; self.check_depth(depth)?; let permit = self.reserve()?; let source_state = self.lookup(&prior)?; @@ -625,35 +609,41 @@ impl Subagents { let (reply, response) = oneshot::channel(); let manager = self.clone(); - tokio::spawn(async move { - let source_state = Arc::clone(&operation.source_state); - let reservation = operation.id.clone(); - let result = manager.run_fork(operation, &reply).await; - manager.finish_forking(&source_state, &reservation).await; - manager - .handoff_fork_result(&parent_session_id, reply, result) - .await; - }); - let response = response.await.map_err(|_| { + tokio::spawn( + async move { + let source_state = Arc::clone(&operation.source_state); + let reservation = operation.id.clone(); + let result = manager.run_fork(operation, &reply).await; + manager.finish_forking(&source_state, &reservation).await; + match result { + Ok(value) => manager.handoff_fork_success(reply, value).await, + Err(error) => { + let _ = reply.send(Err(error)); + } + } + } + .instrument(tracing::Span::current()), + ); + match response.await.map_err(|_| { ChildError::Failed("subagent fork task stopped before returning a result".into()) - })?; - if response.acknowledge.send(()).is_err() { - return Err(match response.value { - Err(error) => error, - Ok(_) => ChildError::Failed( - "subagent fork task stopped before transferring ownership".into(), - ) - .with_effects(response.effects), - }); + })? { + Ok(success) => { + success.acknowledge.send(()).map_err(|_| { + ChildError::Failed( + "subagent fork task stopped before transferring ownership".into(), + ) + })?; + Ok(success.value) + } + Err(error) => Err(error), } - response.value } async fn run_fork( &self, operation: ForkOperation, - reply: &oneshot::Sender, - ) -> Result<(SubagentValue, crate::effects::PossibleEffects), ChildError> { + reply: &oneshot::Sender>, + ) -> Result { let ForkOperation { source_id, source_state, @@ -803,24 +793,18 @@ impl Subagents { .await); } }; - let effects = output.possible_effects(); let (output, updates) = turn_output(output, contract.as_deref()); let mut locked = state.lock().await; if reply.is_closed() { drop(locked); return Err(self - .cleanup_installed_child( - &id, - &state, - &child, - ChildError::Cancelled.with_effects(effects), - ) + .cleanup_installed_child(&id, &state, &child, ChildError::Cancelled) .await); } if let Err(error) = self.check_active(&locked) { drop(locked); return Err(self - .cleanup_installed_child(&id, &state, &child, error.with_effects(effects)) + .cleanup_installed_child(&id, &state, &child, error) .await); } locked.status = SubagentStatus::Idle; @@ -832,16 +816,13 @@ impl Subagents { let event = locked.runtime_event(id.clone()); drop(locked); self.emit_event(event); - Ok(( - SubagentValue { - id, - name: Some(name), - output, - generation, - updates, - }, - effects, - )) + Ok(SubagentValue { + id, + name: Some(name), + output, + generation, + updates, + }) } async fn list( @@ -992,32 +973,16 @@ impl Subagents { }) } - async fn handoff_fork_result( + async fn handoff_fork_success( &self, - parent_session_id: &str, - reply: oneshot::Sender, - result: Result<(SubagentValue, crate::effects::PossibleEffects), ChildError>, + reply: oneshot::Sender>, + value: SubagentValue, ) { - let (cleanup, effects) = match &result { - Ok((value, effects)) => (Some(value.clone()), *effects), - Err(error) => (None, error.possible_effects()), - }; + let cleanup = value.clone(); let (acknowledge, acknowledged) = oneshot::channel(); - let sent = reply - .send(ForkReply { - value: result.map(|(value, _)| value), - effects, - acknowledge, - }) - .is_ok(); - // Both outcomes need acknowledgment: send can succeed while the caller - // drops its future before receiving the reply. Only an acknowledged - // caller owns failure recording through result(). + let sent = reply.send(Ok(ForkSuccess { value, acknowledge })).is_ok(); if !sent || acknowledged.await.is_err() { - if let Some(value) = cleanup { - self.cleanup_abandoned_fork(&value).await; - } - record_child_failure(parent_session_id, effects); + self.cleanup_abandoned_fork(&cleanup).await; } } @@ -1074,10 +1039,11 @@ impl Subagents { match child.close().await { Ok(()) => error, Err(cleanup) if child_error_is_terminal(&cleanup, &child) => error, - Err(_) => { + Err(cleanup) => { Self::watch_permit_until_process_exit(permit, &child); - tracing::warn!("failed to clean up retired subagent session"); - error + ChildError::Failed(format!( + "{error}; failed to clean up retired subagent session: {cleanup}" + )) } } } @@ -1108,10 +1074,11 @@ impl Subagents { match child.close().await { Ok(()) => error, Err(cleanup) if child_error_is_terminal(&cleanup, child) => error, - Err(_) => { + Err(cleanup) => { self.retain_permit_until_process_exit(state, child).await; - tracing::warn!("failed to clean up retired subagent session"); - error + ChildError::Failed(format!( + "{error}; failed to clean up retired subagent session: {cleanup}" + )) } } } @@ -1124,18 +1091,19 @@ impl Subagents { error: ChildError, ) -> ChildError { let manager = self.clone(); - let fallback = error.clone(); - match tokio::spawn(async move { - manager - .cleanup_installed_child(&id, &state, &child, error) - .await - }) + match tokio::spawn( + async move { + manager + .cleanup_installed_child(&id, &state, &child, error) + .await + } + .instrument(tracing::Span::current()), + ) .await { Ok(error) => error, - Err(_) => { - tracing::warn!("retired subagent cleanup task failed"); - fallback + Err(error) => { + ChildError::Failed(format!("retired subagent cleanup task failed: {error}")) } } } @@ -1498,29 +1466,15 @@ fn cancellation(context: &ToolContext<'_>) -> TurnCancellation { .map(|value| value.handle().checkpoint()) .unwrap_or_default() } -fn tool_failure(error: &ChildError) -> ToolError { - match error.root() { - ChildError::Cancelled | ChildError::TerminalCancelled => ToolError::Cancelled, - ChildError::Failed(message) | ChildError::TerminalFailed(message) => { - ToolError::ExecutionFailed(message.clone()) - } - ChildError::Observed { .. } => unreachable!("root unwraps observations"), - } -} - -fn record_child_failure(session_id: &str, effects: crate::effects::PossibleEffects) { - if let Err(log_error) = crate::fatal::record_child_failure(session_id, effects) { - tracing::warn!(%log_error, "could not store child failure observations"); - } -} - fn result( request: ToolRequest, value: Result, ) -> Result { - let value = value.map_err(|error| { - record_child_failure(&request.session_id.0, error.possible_effects()); - tool_failure(&error) + let value = value.map_err(|error| match error { + ChildError::Cancelled | ChildError::TerminalCancelled => ToolError::Cancelled, + ChildError::Failed(error) | ChildError::TerminalFailed(error) => { + ToolError::ExecutionFailed(error) + } })?; Ok(ToolResult::new(ToolResultPart::success( request.call_id, @@ -1659,20 +1613,16 @@ impl Tool for ForkTool { let input: ForkInput = serde_json::from_value(request.input.clone()) .map_err(|e| ToolError::InvalidInput(e.to_string()))?; let contract = input.output_schema.map(OutputContract::new).transpose()?; - let parent_session_id = request.session_id.0.clone(); result( request, self.manager .fork( - parent_session_id, - ForkRequest { - prior: input.subagent, - prompt: input.prompt, - name: input.name, - contract: contract.map(Arc::new), - }, + input.subagent, + input.prompt, + input.name, self.depth, cancellation(context), + contract.map(Arc::new), ) .await, ) diff --git a/src/tools/subagent/tests.rs b/src/tools/subagent/tests.rs index abfa8324..d3a62a87 100644 --- a/src/tools/subagent/tests.rs +++ b/src/tools/subagent/tests.rs @@ -625,15 +625,12 @@ async fn create_uses_requested_working_directory_without_changing_parent() { let branch = manager .fork( - session::new_id(), - super::ForkRequest { - prior: source.clone(), - prompt: "MOCK_CWD".into(), - name: None, - contract: None, - }, + source.clone(), + "MOCK_CWD".into(), + None, 0, TurnCancellation::default(), + None, ) .await .unwrap(); @@ -847,15 +844,12 @@ impl MockAcpScenario { tokio::spawn(async move { manager .fork( - session::new_id(), - super::ForkRequest { - prior: source, - prompt: prompt.into(), - name: None, - contract: None, - }, + source, + prompt.into(), + None, 0, TurnCancellation::default(), + None, ) .await }) @@ -1203,15 +1197,12 @@ async fn failed_create_and_fork_startup_record_failed_removed_transitions() { assert!( failed_fork .fork( - session::new_id(), - super::ForkRequest { - prior: source, - prompt: "fork".into(), - name: None, - contract: None, - }, + source, + "fork".into(), + None, 0, TurnCancellation::default(), + None, ) .await .is_err() @@ -1359,15 +1350,12 @@ async fn native_fork_releases_the_source_before_the_branch_prompt() { let fork_error = scenario .manager .fork( - session::new_id(), - super::ForkRequest { - prior: source.clone(), - prompt: "second branch".into(), - name: None, - contract: None, - }, + source.clone(), + "second branch".into(), + None, 0, TurnCancellation::default(), + None, ) .await .unwrap_err(); @@ -1417,42 +1405,19 @@ async fn native_fork_releases_the_source_before_the_branch_prompt() { #[tokio::test] async fn dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit() { - if crate::effects::isolated_test( - "tools::subagent::tests::dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit", - ) { - return; - } let scenario = MockAcpScenario::new(ScenarioOptions { - gate_prompt: Some("MOCK_RICH_OUTPUT"), + gate_prompt: Some("branch"), fail_close_session: Some("branch-1"), ..Default::default() }); let source = scenario.create("source").await; - let parent_session_id = session::new_id(); - let fork_manager = scenario.manager.clone(); - let fork_source = source.clone(); - let parent = parent_session_id.clone(); - let fork = tokio::spawn(async move { - fork_manager - .fork( - parent, - super::ForkRequest { - prior: fork_source, - prompt: "MOCK_RICH_OUTPUT".into(), - name: None, - contract: None, - }, - 0, - TurnCancellation::default(), - ) - .await - }); + let fork = scenario.spawn_fork(source.clone(), "branch"); scenario .wait_for(|request| { matches!( request, LoggedRequest::Prompt { session_id, text } - if session_id == "branch-1" && text == "MOCK_RICH_OUTPUT" + if session_id == "branch-1" && text == "branch" ) }) .await; @@ -1491,9 +1456,6 @@ async fn dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit [source.id.as_str()] ); - wait_for_fork_diagnostic(&parent_session_id).await; - assert_rich_fork_diagnostic(&parent_session_id); - scenario .manager .close(&source.id, &TurnCancellation::default()) @@ -1502,217 +1464,30 @@ async fn dropped_fork_with_failed_close_holds_only_its_permit_until_process_exit wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; } -fn fork_diagnostic_count(session_id: &str) -> usize { - let directory = PathBuf::from(std::env::var_os("HOME").unwrap()) - .join(".kit/errors") - .join(session_id); - std::fs::read_dir(directory) - .into_iter() - .flatten() - .filter_map(Result::ok) - .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) - .count() -} - -async fn wait_for_fork_diagnostic(session_id: &str) { - tokio::time::timeout(std::time::Duration::from_secs(3), async { - while fork_diagnostic_count(session_id) == 0 { - tokio::task::yield_now().await; - } - }) - .await - .expect("detached fork did not retain observations"); -} - -fn assert_rich_fork_diagnostic(session_id: &str) { - assert_eq!(fork_diagnostic_count(session_id), 1); - let record = crate::effects::test_record(session_id); - assert_eq!(record["session_id"], session_id); - assert_eq!(record["surface"], "subagent"); - assert_eq!(record["code"], "subagent_failed"); - assert_eq!( - record["possible_effects"], - json!({ - "source": "acp_notifications", - "assistant_output_observed": true, - "tool_emission_observed": true, - "tool_execution_start_reported": false, - "tool_execution_completion_reported": true, - "observation_incomplete": true, - }) - ); - let encoded = record.to_string(); - for private in ["rich done", "call-1", "Inspect files", "MOCK_RICH_OUTPUT"] { - assert!(!encoded.contains(private)); - } -} - -#[tokio::test] -async fn fork_handoff_retains_observations_only_when_caller_abandons_delivery() { - if crate::effects::isolated_test( - "tools::subagent::tests::fork_handoff_retains_observations_only_when_caller_abandons_delivery", - ) { - return; - } - // Exercise closed delivery, dropped acknowledgment, and normal delivery for - // both outcomes. Positive facts come from actual ACP notifications first. - for failed in [false, true] { - for delivery in ["closed", "unacknowledged", "acknowledged"] { - let scenario = MockAcpScenario::new(ScenarioOptions::default()); - let branch = scenario.create("branch").await; - let state = scenario.manager.lookup(&branch).unwrap(); - let child = state.lock().await.child.clone().unwrap(); - let effects = child - .prompt("MOCK_RICH_OUTPUT".into(), TurnCancellation::default()) - .await - .unwrap() - .possible_effects(); - assert!(effects.tool_execution_completion_reported); - let outcome = if failed { - Err(scenario - .manager - .cleanup_installed_child( - &branch.id, - &state, - &child, - ChildError::Cancelled.with_effects(effects), - ) - .await) - } else { - Ok((branch.clone(), effects)) - }; - let parent_session_id = session::new_id(); - let parent = parent_session_id.clone(); - let (reply, response) = oneshot::channel(); - let manager = scenario.manager.clone(); - let response = if delivery == "closed" { - drop(response); - None - } else { - Some(response) - }; - let handoff = tokio::spawn(async move { - manager.handoff_fork_result(&parent, reply, outcome).await; - }); - let mut delivered = None; - if let Some(response) = response { - let receipt = response.await.unwrap(); - assert_eq!(receipt.effects, effects); - if delivery == "acknowledged" { - receipt.acknowledge.send(()).unwrap(); - delivered = Some(receipt.value); - } - } - handoff.await.unwrap(); - if let Some(value) = delivered { - // The detached owner must not duplicate the normal result writer. - assert_eq!(fork_diagnostic_count(&parent_session_id), 0); - let request = ToolRequest::new( - agentkit_core::ToolCallId::new("fork-call"), - ToolName::new("fork"), - json!({}), - agentkit_core::SessionId::new(parent_session_id.clone()), - agentkit_core::TurnId::new("fork-turn"), - ); - let result = result(request, value); - if failed { - assert!(matches!(result, Err(ToolError::Cancelled))); - assert_rich_fork_diagnostic(&parent_session_id); - } else { - assert!(result.is_ok()); - assert_eq!(fork_diagnostic_count(&parent_session_id), 0); - scenario - .manager - .close(&branch.id, &TurnCancellation::default()) - .await - .unwrap(); - } - } else { - assert_rich_fork_diagnostic(&parent_session_id); - } - assert!( - scenario - .manager - .list(&TurnCancellation::default()) - .await - .unwrap() - .is_empty() - ); - // The fixture's extra State reference still owns the process permit. - drop(child); - drop(state); - wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; - } - } -} - #[tokio::test] -async fn dropped_fork_after_positive_observations_retains_diagnostic() { - if crate::effects::isolated_test( - "tools::subagent::tests::dropped_fork_after_positive_observations_retains_diagnostic", - ) { - return; - } - let mut scenario = MockAcpScenario::new(ScenarioOptions { - gate_prompt: Some("MOCK_RICH_OUTPUT"), - ..Default::default() +async fn successful_fork_handoff_cleans_up_if_receipt_is_not_acknowledged() { + let scenario = MockAcpScenario::new(ScenarioOptions::default()); + let branch = scenario.create("branch").await; + let (reply, response) = oneshot::channel(); + let manager = scenario.manager.clone(); + let cleanup_branch = branch.clone(); + let handoff = tokio::spawn(async move { + manager.handoff_fork_success(reply, cleanup_branch).await; }); - let (manager, events) = observe_events(scenario.manager.clone()); - scenario.manager = manager; - let source = scenario.create("source").await; - let parent_session_id = session::new_id(); - let mut fork = Box::pin(scenario.manager.fork( - parent_session_id.clone(), - super::ForkRequest { - prior: source.clone(), - prompt: "MOCK_RICH_OUTPUT".into(), - name: None, - contract: None, - }, - 0, - TurnCancellation::default(), - )); - tokio::select! { - _ = &mut fork => panic!("gated fork returned early"), - _ = scenario.wait_for(|request| matches!(request, - LoggedRequest::Prompt { text, .. } if text == "MOCK_RICH_OUTPUT")) => {} - } - MockAcpScenario::release(&scenario.prompt_release); - // Leave the caller unpolled until run_fork has observed the child output - // and published success. Its queued reply has not been acknowledged. - tokio::time::timeout(std::time::Duration::from_secs(3), async { - loop { - if emitted(&events).iter().any(|event| { - matches!(event, - events::RuntimeEvent::SubagentStateChanged { - id, status: SubagentStatus::Idle, .. - } if id != &source.id) - }) { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("fork did not observe successful child output"); - assert_eq!(fork_diagnostic_count(&parent_session_id), 0); - drop(fork); - wait_for_fork_diagnostic(&parent_session_id).await; - assert_rich_fork_diagnostic(&parent_session_id); - let listed = scenario - .manager - .list(&TurnCancellation::default()) - .await - .unwrap(); - assert_eq!( - listed.iter().map(listing_id).collect::>(), - [source.id.as_str()] + + let success = response.await.unwrap().unwrap(); + assert_eq!(success.value.id, branch.id); + drop(success); + handoff.await.unwrap(); + + assert!( + scenario + .manager + .list(&TurnCancellation::default()) + .await + .unwrap() + .is_empty() ); - scenario - .manager - .close(&source.id, &TurnCancellation::default()) - .await - .unwrap(); wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; } @@ -2005,15 +1780,12 @@ async fn fork_uses_its_fresh_preferred_name() { let fork = manager .fork( - session::new_id(), - super::ForkRequest { - prior: source, - prompt: "branch".into(), - name: Some("Reviewer".into()), - contract: None, - }, + source, + "branch".into(), + Some("Reviewer".into()), 0, TurnCancellation::default(), + None, ) .await .unwrap(); @@ -2068,15 +1840,12 @@ async fn generic_harness_without_native_fork_returns_unsupported() { let error = manager .fork( - session::new_id(), - super::ForkRequest { - prior, - prompt: "branch".into(), - name: None, - contract: None, - }, + prior, + "branch".into(), + None, 0, TurnCancellation::default(), + None, ) .await .unwrap_err(); @@ -2086,130 +1855,3 @@ async fn generic_harness_without_native_fork_returns_unsupported() { "ACP harness \"acp.generic\" does not advertise session/fork; transcript fallback is only available for Kit" ); } - -#[tokio::test] -async fn observed_failures_keep_terminal_and_cancellation_classification() { - let (child, _) = ChildSession::closure_probe_for_test(); - let effects = crate::effects::PossibleEffects { - assistant_output_observed: true, - ..crate::effects::PossibleEffects::default() - }; - for error in [ - ChildError::TerminalCancelled, - ChildError::TerminalFailed("transport ended".into()), - ] { - let observed = ChildError::Observed { - error: Box::new(error), - effects, - }; - assert!(child_error_is_terminal(&observed, &child)); - assert_eq!(observed.possible_effects(), effects); - match observed.root() { - ChildError::TerminalCancelled => { - assert!(matches!(tool_failure(&observed), ToolError::Cancelled)) - } - _ => assert!( - matches!(tool_failure(&observed), ToolError::ExecutionFailed(message) if message == "transport ended") - ), - } - } - let observed = ChildError::Observed { - error: Box::new(ChildError::Failed("refused".into())), - effects, - }; - assert!(!child_error_is_terminal(&observed, &child)); - assert!( - matches!(tool_failure(&observed), ToolError::ExecutionFailed(message) if message == "refused") - ); -} - -#[tokio::test] -async fn successful_prompt_after_retirement_keeps_observed_effects() { - let scenario = MockAcpScenario::new(ScenarioOptions { - gate_prompt: Some("MOCK_RICH_OUTPUT"), - ..Default::default() - }); - let source = scenario.create("source").await; - let prompt_manager = scenario.manager.clone(); - let prompt_source = source.clone(); - let prompt = tokio::spawn(async move { - prompt_manager - .prompt( - prompt_source, - "MOCK_RICH_OUTPUT".into(), - TurnCancellation::default(), - None, - ) - .await - }); - scenario - .wait_for(|request| { - matches!(request, LoggedRequest::Prompt { text, .. } if text == "MOCK_RICH_OUTPUT") - }) - .await; - scenario - .manager - .close(&source.id, &TurnCancellation::default()) - .await - .unwrap(); - MockAcpScenario::release(&scenario.prompt_release); - let error = prompt.await.unwrap().unwrap_err(); - let effects = error.possible_effects(); - assert!(effects.assistant_output_observed); - assert!(effects.tool_emission_observed); - assert!(effects.tool_execution_completion_reported); - assert!(effects.observation_incomplete); - assert!( - scenario - .manager - .list(&TurnCancellation::default()) - .await - .unwrap() - .is_empty() - ); -} - -#[tokio::test] -async fn failed_cleanup_preserves_cancellation_and_holds_capacity() { - for original in [ChildError::Cancelled, ChildError::TerminalCancelled] { - let scenario = MockAcpScenario::new(ScenarioOptions { - fail_close_session: Some("branch-1"), - ..Default::default() - }); - let source = scenario.create("source").await; - let branch = scenario - .spawn_fork(source.clone(), "branch") - .await - .unwrap() - .unwrap(); - let state = scenario.manager.lookup(&branch).unwrap(); - let child = state.lock().await.child.clone().unwrap(); - let effects = crate::effects::PossibleEffects { - assistant_output_observed: true, - ..Default::default() - }; - let terminal = matches!(original, ChildError::TerminalCancelled); - let error = scenario - .manager - .cleanup_installed_child(&branch.id, &state, &child, original.with_effects(effects)) - .await; - assert_eq!( - matches!(error.root(), ChildError::TerminalCancelled), - terminal - ); - assert!(matches!(tool_failure(&error), ToolError::Cancelled)); - assert_eq!(error.possible_effects(), effects); - assert_eq!( - scenario.manager.capacity.available_permits(), - MAX_LIVE_SUBAGENTS - 2 - ); - drop(state); - drop(child); - scenario - .manager - .close(&source.id, &TurnCancellation::default()) - .await - .unwrap(); - wait_for_available_permits(&scenario.manager, MAX_LIVE_SUBAGENTS).await; - } -} diff --git a/src/tui/mod.rs b/src/tui/mod.rs index f06fdce4..d4c33ab3 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -4105,6 +4105,8 @@ a = [still text] assert_eq!( args, [ + "--internal-capture-error-spans", + "false", "--otel-endpoint", "http://collector:4318/v1/traces", "--otel-protocol", diff --git a/tests/runtime.rs b/tests/runtime.rs index f12f433f..d49cdaa5 100644 --- a/tests/runtime.rs +++ b/tests/runtime.rs @@ -629,8 +629,8 @@ async fn execute_compose_cancelled( script: &str, cancellation: Option, ) -> ToolExecutionOutcome { - // Artifact paths are HOME/session/call scoped. Parallel tests must not - // remove a spill directory still in use by another invocation. + // HOME/session/call-scoped artifacts must not share a spill directory with + // another parallel invocation that may remove it during cleanup. static NEXT_CALL: AtomicUsize = AtomicUsize::new(0); let call_id = ToolCallId::new(format!( "compose-test-{}", From 3156f58e440a597775bac32b24fa9981743105f4 Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 23:16:16 +0100 Subject: [PATCH 7/7] docs: explain when to enable error span capture --- .../user/getting-started-and-configuration.md | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/docs/user/getting-started-and-configuration.md b/docs/user/getting-started-and-configuration.md index fa2f642a..24c0a8dc 100644 --- a/docs/user/getting-started-and-configuration.md +++ b/docs/user/getting-started-and-configuration.md @@ -211,30 +211,13 @@ For settings exposed by a command, precedence is: 2. values in `~/.kit/config.toml`; 3. built-in defaults. -Set `capture_error_spans = true` to include a bounded structured span history -alongside existing fatal errors in `~/.kit/errors//`. It defaults to -`false` when omitted. This local collector is independent of OTLP export: -neither setting enables the other. Disabled collection installs no diagnostic -layer and retains no diagnostic span history. Built-in TUI and `acp.kit` children -inherit the resolved setting and must use a compatible Kit executable. - -The optional `span_context` field contains operation-local parent indexes and -allowlisted operation names, launch kinds, tool-error classifications, booleans, -and bounded counts from tracing spans. It excludes external identifiers, -`Debug`/`Display` values, messages, prompts, tool arguments/results, URLs, and -provider payloads, even when OTEL message-content capture is enabled. Attributes -set only through OpenTelemetry APIs are not collected. Histories are limited to -24 fragments, eight descendant levels, six fields per fragment, 32 bytes per -string value, and 12 KiB of serialized context. Existing schema-v2 errors remain -readable; files are not rewritten on read. - -Histories cover instrumented prompt and autonomous operation boundaries, including -already-closed child spans. They are partial operation histories, not proof that -sibling spans caused an error or that replay is safe. Collection is best-effort; -missing observations never prove that no effects occurred. Upstream TaskManager -spawns currently lose tracing ancestry, so tool task bodies and background work -inside those spawns can be absent even when their dispatch span was observed. -Remote child-process spans are not transported into the parent's error log. +Set `capture_error_spans = true` when troubleshooting unexpected failures or +preparing a bug report. Kit adds diagnostic context about recent operations to +error logs in `~/.kit/errors//`, which can help explain a failure. + +It is disabled by default to avoid additional collection overhead and does not +require OpenTelemetry. The extra context excludes prompts and tool inputs and +outputs; it is not a complete execution history. The OpenTelemetry endpoint follows the same CLI-over-TOML precedence, then falls back to the standard `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. If none