diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/deletion.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/deletion.rs index 3410f4be0..446c3bec0 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/deletion.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/deletion.rs @@ -321,6 +321,27 @@ pub(crate) fn ensure_run_conversation_writable_with_connection( ensure_conversation_writable_with_connection(conn, &root_session_id) } +/// Resolve exact PR1 ownership before checking a Session's root fence. +/// +/// This is used only by deletion planning, including direct Worker deletion; +/// ordinary submission paths keep their PR2 fast path and never call it. +pub(crate) fn ensure_session_conversation_writable_with_connection( + conn: &Connection, + session_id: &str, +) -> Result<(), String> { + match exact_submission_scope_with_connection(conn, session_id)? { + AgentOrgSubmissionScope::Unknown => Err(format!( + "Agent Org deletion ownership remained unknown for session {session_id}" + )), + AgentOrgSubmissionScope::Ordinary => { + ensure_conversation_writable_with_connection(conn, session_id) + } + AgentOrgSubmissionScope::Run { run_id } => { + ensure_run_conversation_writable_with_connection(conn, &run_id) + } + } +} + /// A Run is writable only while it is running and its root conversation is /// not fenced. Callers that scanned earlier must repeat this check in the /// transaction that performs their durable write. @@ -426,7 +447,6 @@ pub(crate) fn establish_conversation_delete_fence_with_connection( /// Own the short, serialized transaction used to establish a deletion fence. /// Callers that must compare topology inside the same snapshot should instead /// use [`establish_conversation_delete_fence_with_connection`]. -#[cfg(test)] pub(crate) fn establish_conversation_delete_fence(root_session_id: &str) -> Result<(), String> { let root_session_id = root_session_id.to_string(); let outcome = with_sessions_writer(|| -> Result<_, String> { diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs index 96a04d90a..313005726 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs @@ -19,13 +19,11 @@ mod tests; #[cfg(debug_assertions)] #[doc(hidden)] pub use deletion::debug_establish_e2e_conversation_delete_fence; -#[cfg(test)] -pub(crate) use deletion::establish_conversation_delete_fence; pub(crate) use deletion::{ admit_agent_org_submission, admit_known_agent_org_submission, agent_org_submission_in_progress, ensure_conversation_writable_with_connection, - establish_conversation_delete_fence_with_connection, exact_submission_scope, - is_run_writable_with_connection, recheck_agent_org_submission, + ensure_session_conversation_writable_with_connection, establish_conversation_delete_fence, + exact_submission_scope, is_run_writable_with_connection, recheck_agent_org_submission, recheck_known_agent_org_submission, remove_conversation_delete_fence_with_connection, submission_scope_for_loaded_session, AgentOrgSubmissionLease, AgentOrgSubmissionPolicy, AgentOrgSubmissionScope, SharedAgentOrgSubmissionScope, @@ -41,6 +39,7 @@ pub use finality::{ }; pub(crate) use progress::bump_work_revision_in_tx; pub use progress::AgentOrgRunProgress; +pub(crate) use store::AgentOrgRunDeleteOutcome; pub use store::AgentOrgRunStore; pub(crate) use worker::recovery_dispatch_recipient_is_available; pub use worker::{WorkerSessionInfo, WorkerSessionRuntime}; diff --git a/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs b/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs index 53f60e965..2503fe8d4 100644 --- a/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs @@ -33,6 +33,7 @@ pub use crud::{ update_status, update_work_item_link, update_worktree_merge_status, upsert_session, UnifiedSessionRecord, }; +#[allow(unused_imports)] pub(crate) use crud::{ delete_session_with_connection, finish_session_delete, notify_session_upserted, prepare_session_delete, upsert_session_with_connection, diff --git a/src-tauri/crates/agent-core/src/foundation/persistence/db_helpers/mod.rs b/src-tauri/crates/agent-core/src/foundation/persistence/db_helpers/mod.rs index 78b613956..76f1863f4 100644 --- a/src-tauri/crates/agent-core/src/foundation/persistence/db_helpers/mod.rs +++ b/src-tauri/crates/agent-core/src/foundation/persistence/db_helpers/mod.rs @@ -152,7 +152,9 @@ pub fn delete_session_cascade(session_id: &str, tables: &[&str]) -> SqliteResult }) } -/// Transaction-aware form of [`delete_session_cascade`]. +/// Transaction-aware form of [`delete_session_cascade`]. Image files are left +/// for the existing orphan-image housekeeping pass: deleting them here could +/// race an image persisted just before its message row acquires the writer. /// /// The caller owns the transaction boundary. This is used when several Rust /// Agent Org sessions and their run-owned rows must commit or roll back as one @@ -164,20 +166,6 @@ pub(crate) fn delete_session_cascade_with_connection( session_id: &str, tables: &[&str], ) -> SqliteResult<()> { - // Collect image file paths before deleting the rows. Infer the - // prefix from the first table that ends with "_messages". - let prefix = tables - .iter() - .find(|t| t.ends_with("_messages")) - .and_then(|t| t.strip_suffix("_messages")); - - if let Some(prefix) = prefix { - let image_paths = collect_session_image_paths_with_connection(conn, prefix, session_id)?; - if !image_paths.is_empty() { - super::images::delete_image_files(&image_paths); - } - } - delete_session_rows_with_connection(conn, session_id, tables) } diff --git a/src-tauri/crates/agent-core/src/state/commands/session/agent_org_delete.rs b/src-tauri/crates/agent-core/src/state/commands/session/agent_org_delete.rs new file mode 100644 index 000000000..067dcef89 --- /dev/null +++ b/src-tauri/crates/agent-core/src/state/commands/session/agent_org_delete.rs @@ -0,0 +1,1159 @@ +//! Root-conversation deletion for Rust Agent Org runs. +//! +//! The protocol is intentionally split into durable phases: plan and preflight, +//! establish the Root fence, quiesce runtimes, then remove all SQLite-owned +//! state in one transaction. A committed fence survives every later failure. + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +use crate::coordination::agent_org_runs::{ + agent_org_submission_in_progress, ensure_session_conversation_writable_with_connection, + establish_conversation_delete_fence, remove_conversation_delete_fence_with_connection, + AgentOrgRunDeleteOutcome, AgentOrgRunStatus, AgentOrgRunStore, COORDINATOR_MEMBER_ID, +}; +use crate::definitions::orgs::{parse_cli_agent_org_reference, OrgDefinition, OrgMember}; +use crate::session::persistence::{self as session_persistence, session_type}; +use crate::session::SessionStatus; +use crate::state::control_flow::CancelReason; +use crate::state::{AgentAppState, AgentSession}; +use database::db::{get_connection, with_sessions_writer}; +use rusqlite::{params, Connection, OptionalExtension}; + +use super::persistence::DeleteSessionReceipt; + +pub(super) const MAX_AGENT_ORG_DELETE_RUNS: usize = 1_024; +pub(super) const MAX_AGENT_ORG_DELETE_SESSIONS: usize = 1_024; +const AGENT_ORG_DELETE_STOP_TIMEOUT: Duration = Duration::from_secs(10); +const AGENT_ORG_DELETE_STOP_POLL_INTERVAL: Duration = Duration::from_millis(50); +const SESSION_STATE_TABLES: &[&str] = &[ + "session_turns", + "session_turn_index_state", + "sessions", + "goal_loop_state", + "housekeeper_context_compaction", +]; + +fn root_refusal(root: &str, reason: impl std::fmt::Display) -> String { + format!("Refusing to delete Agent Org root {root}: {reason}") +} + +fn run_refusal(run: &str, reason: impl std::fmt::Display) -> String { + format!("Refusing to delete Agent Org run {run}: {reason}") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct AgentOrgSessionDeleteNode { + pub(super) session_id: String, + pub(super) parent_session_id: Option, + pub(super) status: SessionStatus, + pub(super) owning_run_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct AgentOrgRunDeletePlan { + pub(super) run_id: String, + pub(super) status: AgentOrgRunStatus, + pub(super) worker_session_ids: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct AgentOrgSessionDeletePlan { + pub(super) root_session_id: String, + pub(super) runs: Vec, + pub(super) sessions: Vec, +} + +#[derive(Debug)] +struct FencedAgentOrgDeletePlan(AgentOrgSessionDeletePlan); + +#[derive(Debug)] +struct QuiescedAgentOrgDeletePlan { + plan: AgentOrgSessionDeletePlan, + safe_inflight_session_ids: HashSet, +} + +#[derive(Debug)] +struct AgentOrgSessionPostCommitCleanup { + session_id: String, + workspace_path: Option, + managed_worktree: bool, +} + +struct AgentOrgCommittedDelete { + receipt: DeleteSessionReceipt, + run_cleanup: Vec<(String, AgentOrgRunDeleteOutcome)>, + session_cleanup: Vec, +} + +impl AgentOrgCommittedDelete { + fn already_completed(receipt: DeleteSessionReceipt) -> Self { + Self { + receipt, + run_cleanup: Vec::new(), + session_cleanup: Vec::new(), + } + } +} + +#[derive(Debug)] +enum ReloadedAgentOrgDeletePlan { + Present(AgentOrgSessionDeletePlan), + ConcurrentlyCompleted(DeleteSessionReceipt), +} + +async fn reload_agent_org_delete_plan( + root_session_id: String, + expected: AgentOrgSessionDeletePlan, + disappearance: &'static str, +) -> Result { + tokio::task::spawn_blocking(move || { + let conn = get_connection().map_err(|err| err.to_string())?; + match load_agent_org_session_delete_plan(&conn, &root_session_id)? { + Some(plan) => Ok(ReloadedAgentOrgDeletePlan::Present(plan)), + None => completed_agent_org_delete_receipt(&conn, &expected)? + .map(ReloadedAgentOrgDeletePlan::ConcurrentlyCompleted) + .ok_or_else(|| { + root_refusal( + &root_session_id, + format!("ownership disappeared {disappearance}"), + ) + }), + } + }) + .await + .map_err(|err| format!("Agent Org deletion planning worker failed: {err}"))? +} + +pub(super) async fn delete_session( + state: &AgentAppState, + session_id: String, +) -> Result { + let planned_session_id = session_id.clone(); + let initial_plan = tokio::task::spawn_blocking(move || { + let conn = get_connection().map_err(|err| err.to_string())?; + load_agent_org_session_delete_plan(&conn, &planned_session_id) + }) + .await + .map_err(|err| format!("session deletion planning worker failed: {err}"))??; + + let Some(initial_plan) = initial_plan else { + let deleted_session_id = session_id.clone(); + tokio::task::spawn_blocking(move || { + session_persistence::delete_session(&deleted_session_id).map_err(|err| err.to_string()) + }) + .await + .map_err(|err| format!("session deletion worker failed: {err}"))??; + return Ok(DeleteSessionReceipt { + deleted_session_ids: vec![session_id], + }); + }; + + preflight_agent_org_delete_resources(&initial_plan)?; + + let root_session_id = initial_plan.root_session_id.clone(); + tokio::task::spawn_blocking(move || establish_conversation_delete_fence(&root_session_id)) + .await + .map_err(|err| format!("Agent Org deletion fence worker failed: {err}"))??; + + let root_session_id = initial_plan.root_session_id.clone(); + let completion_plan = initial_plan.clone(); + let fenced_plan = + reload_agent_org_delete_plan(root_session_id, completion_plan, "after fencing").await?; + + let fenced_plan = match fenced_plan { + ReloadedAgentOrgDeletePlan::Present(plan) => plan, + ReloadedAgentOrgDeletePlan::ConcurrentlyCompleted(receipt) => { + return finish_agent_org_delete( + state, + AgentOrgCommittedDelete::already_completed(receipt), + ) + .await; + } + }; + + preflight_agent_org_delete_resources(&fenced_plan)?; + let fenced_plan = FencedAgentOrgDeletePlan(fenced_plan); + let safe_inflight_session_ids = stop_agent_org_runtime_sessions(state, &fenced_plan.0).await?; + + let root_session_id = fenced_plan.0.root_session_id.clone(); + let completion_plan = fenced_plan.0.clone(); + let current_plan = + reload_agent_org_delete_plan(root_session_id, completion_plan, "while stopping").await?; + let current_plan = match current_plan { + ReloadedAgentOrgDeletePlan::Present(plan) => plan, + ReloadedAgentOrgDeletePlan::ConcurrentlyCompleted(receipt) => { + return finish_agent_org_delete( + state, + AgentOrgCommittedDelete::already_completed(receipt), + ) + .await; + } + }; + if !agent_org_delete_topology_matches(&fenced_plan.0, ¤t_plan) { + return Err(root_refusal( + &fenced_plan.0.root_session_id, + "ownership changed while stopping", + )); + } + + validate_agent_org_delete_ready(¤t_plan, &safe_inflight_session_ids)?; + ensure_agent_org_runtime_sessions_idle(state, ¤t_plan).await?; + let quiesced_plan = QuiescedAgentOrgDeletePlan { + plan: current_plan, + safe_inflight_session_ids, + }; + + let committed_delete = tokio::task::spawn_blocking(move || { + commit_agent_org_session_hierarchy( + &quiesced_plan.plan, + &quiesced_plan.safe_inflight_session_ids, + ) + }) + .await + .map_err(|err| format!("Agent Org session deletion worker failed: {err}"))??; + + finish_agent_org_delete(state, committed_delete).await +} + +async fn finish_agent_org_delete( + state: &AgentAppState, + committed_delete: AgentOrgCommittedDelete, +) -> Result { + let AgentOrgCommittedDelete { + receipt, + run_cleanup, + session_cleanup, + } = committed_delete; + // Remove in-memory entry points immediately after the durable commit. + // Filesystem cleanup can be slower (notably Git worktree pruning) and + // must not leave a deleted runtime addressable during that interval. + state.remove_sessions(&receipt.deleted_session_ids).await; + if let Some(app_handle) = state.app_handle.as_ref() { + for deleted_session_id in &receipt.deleted_session_ids { + crate::bus::event_pipeline_bridge::evict_session(app_handle, deleted_session_id); + } + } + if !run_cleanup.is_empty() || !session_cleanup.is_empty() { + if let Err(error) = tokio::task::spawn_blocking(move || { + finish_agent_org_post_commit_resources(run_cleanup, session_cleanup) + }) + .await + { + tracing::warn!( + error = %error, + "Agent Org deletion committed, but post-commit resource cleanup worker failed" + ); + } + } + Ok(receipt) +} + +fn completed_agent_org_delete_receipt( + conn: &Connection, + expected_plan: &AgentOrgSessionDeletePlan, +) -> Result, String> { + let fence_or_run_exists = conn + .query_row( + "SELECT EXISTS( + SELECT 1 FROM agent_org_conversation_delete_fences + WHERE root_session_id=?1 + ) OR EXISTS( + SELECT 1 FROM agent_org_runs WHERE root_session_id=?1 + )", + [&expected_plan.root_session_id], + |row| row.get::<_, bool>(0), + ) + .map_err(|err| err.to_string())?; + if fence_or_run_exists { + return Ok(None); + } + for node in &expected_plan.sessions { + let exists = conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM agent_sessions WHERE session_id=?1)", + [&node.session_id], + |row| row.get::<_, bool>(0), + ) + .map_err(|err| err.to_string())?; + if exists { + return Ok(None); + } + } + Ok(Some(DeleteSessionReceipt { + deleted_session_ids: expected_plan + .sessions + .iter() + .map(|node| node.session_id.clone()) + .collect(), + })) +} + +fn preflight_agent_org_delete_resources(plan: &AgentOrgSessionDeletePlan) -> Result<(), String> { + for node in &plan.sessions { + crate::tools::impls::coding::exec::shell_replay::ensure_session_replays_deletable( + &node.session_id, + )?; + } + Ok(()) +} + +pub(super) fn load_agent_org_session_delete_plan( + conn: &Connection, + root_session_id: &str, +) -> Result, String> { + let run_rows = load_root_runs(conn, root_session_id)?; + if run_rows.is_empty() { + // A retained fence means an earlier deletion reached its durable + // boundary but did not finish. Treating this as an ordinary Session + // would strand the fence and bypass the Agent Org retry protocol. + ensure_session_conversation_writable_with_connection(conn, root_session_id)?; + return Ok(None); + } + + let (root_parent_session_id, root_status_raw): (Option, String) = conn + .query_row( + "SELECT parent_session_id, status + FROM agent_sessions + WHERE session_id=?1", + [root_session_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|err| err.to_string())? + .ok_or_else(|| root_refusal(root_session_id, "root session is missing"))?; + let root_status = parse_session_status(root_session_id, &root_status_raw)?; + + validate_descendant_shape(conn, root_session_id)?; + reject_historical_cli_descendants(conn, root_session_id)?; + + let run_ids = run_rows + .iter() + .map(|(run_id, _)| run_id.clone()) + .collect::>(); + let mut coordinator_runs = HashSet::new(); + let mut worker_ids = HashSet::new(); + let mut workers_by_run = HashMap::>::new(); + let mut worker_nodes = Vec::new(); + let mut mapping_count = 0usize; + + let mut stmt = conn + .prepare( + "SELECT mapping.org_run_id, + mapping.member_id, + mapping.session_id, + mapping.role, + session.parent_session_id, + session.status, + session.agent_definition_id, + session.org_member_id + FROM agent_org_run_sessions mapping + JOIN agent_org_runs run ON run.id=mapping.org_run_id + LEFT JOIN agent_sessions session ON session.session_id=mapping.session_id + WHERE run.root_session_id=?1 + ORDER BY mapping.org_run_id, mapping.role, mapping.member_id + LIMIT ?2", + ) + .map_err(|err| err.to_string())?; + let rows = stmt + .query_map( + params![ + root_session_id, + (MAX_AGENT_ORG_DELETE_RUNS + MAX_AGENT_ORG_DELETE_SESSIONS + 1) as i64 + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + )) + }, + ) + .map_err(|err| err.to_string())?; + + for row in rows { + let ( + run_id, + member_id, + session_id, + role, + parent_session_id, + status_raw, + agent_definition_id, + org_member_id, + ) = row.map_err(|err| err.to_string())?; + mapping_count += 1; + if mapping_count > MAX_AGENT_ORG_DELETE_RUNS + MAX_AGENT_ORG_DELETE_SESSIONS { + return Err(root_refusal( + root_session_id, + "ownership exceeds the bounded deletion limit", + )); + } + if !run_ids.contains(&run_id) { + return Err(root_refusal( + root_session_id, + format!("mapping references unexpected run {run_id}"), + )); + } + match role.as_str() { + "coordinator" => { + if member_id != COORDINATOR_MEMBER_ID || session_id != root_session_id { + return Err(run_refusal(&run_id, "invalid Coordinator mapping")); + } + if !coordinator_runs.insert(run_id.clone()) { + return Err(run_refusal(&run_id, "duplicate Coordinator mapping")); + } + } + "worker" => { + let status_raw = status_raw.ok_or_else(|| { + run_refusal( + &run_id, + format!("mapped Worker session {session_id} is missing"), + ) + })?; + if member_id == COORDINATOR_MEMBER_ID + || parent_session_id.as_deref() != Some(root_session_id) + || agent_definition_id.as_deref().is_none_or(str::is_empty) + || org_member_id.as_deref() != Some(member_id.as_str()) + { + return Err(run_refusal( + &run_id, + format!("mapped Worker session {session_id} has inconsistent identity"), + )); + } + if !worker_ids.insert(session_id.clone()) { + return Err(root_refusal( + root_session_id, + format!("Worker session {session_id} has duplicate ownership"), + )); + } + let status = parse_session_status(&session_id, &status_raw)?; + workers_by_run + .entry(run_id.clone()) + .or_default() + .push(session_id.clone()); + worker_nodes.push(AgentOrgSessionDeleteNode { + session_id, + parent_session_id, + status, + owning_run_id: Some(run_id), + }); + } + _ => { + return Err(run_refusal( + &run_id, + format!("unknown ownership role {role:?}"), + )); + } + } + } + + for (run_id, _) in &run_rows { + if !coordinator_runs.contains(run_id) { + return Err(run_refusal(run_id, "Coordinator mapping is missing")); + } + } + if worker_nodes.len() + 1 > MAX_AGENT_ORG_DELETE_SESSIONS { + return Err(root_refusal( + root_session_id, + format!("exact Session ownership exceeds {MAX_AGENT_ORG_DELETE_SESSIONS} nodes"), + )); + } + reject_unmapped_rust_workers(conn, root_session_id)?; + + worker_nodes.sort_by(|left, right| left.session_id.cmp(&right.session_id)); + let mut runs = run_rows + .into_iter() + .map(|(run_id, status)| { + let mut worker_session_ids = workers_by_run.remove(&run_id).unwrap_or_default(); + worker_session_ids.sort(); + AgentOrgRunDeletePlan { + run_id, + status, + worker_session_ids, + } + }) + .collect::>(); + runs.sort_by(|left, right| left.run_id.cmp(&right.run_id)); + worker_nodes.push(AgentOrgSessionDeleteNode { + session_id: root_session_id.to_string(), + parent_session_id: root_parent_session_id, + status: root_status, + owning_run_id: None, + }); + + Ok(Some(AgentOrgSessionDeletePlan { + root_session_id: root_session_id.to_string(), + runs, + sessions: worker_nodes, + })) +} + +fn load_root_runs( + conn: &Connection, + root_session_id: &str, +) -> Result, String> { + let mut stmt = conn + .prepare( + "SELECT id, status, org_snapshot_json + FROM agent_org_runs + WHERE root_session_id=?1 + ORDER BY id + LIMIT ?2", + ) + .map_err(|err| err.to_string())?; + let rows = stmt + .query_map( + params![root_session_id, (MAX_AGENT_ORG_DELETE_RUNS + 1) as i64], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + }, + ) + .map_err(|err| err.to_string())?; + let rows = rows + .collect::, _>>() + .map_err(|err| err.to_string())?; + if rows.len() > MAX_AGENT_ORG_DELETE_RUNS { + return Err(root_refusal( + root_session_id, + format!("Run ownership exceeds {MAX_AGENT_ORG_DELETE_RUNS} rows"), + )); + } + rows.into_iter() + .map(|(run_id, status_raw, snapshot_json)| { + if snapshot_json + .as_deref() + .map(run_snapshot_contains_cli) + .transpose()? + .unwrap_or(false) + { + return Err(run_refusal(&run_id, "CLI members are unsupported")); + } + let status = AgentOrgRunStatus::parse(&status_raw).ok_or_else(|| { + run_refusal(&run_id, format!("unknown run status {status_raw:?}")) + })?; + Ok((run_id, status)) + }) + .collect() +} + +fn run_snapshot_contains_cli(snapshot_json: &str) -> Result { + fn members_contain_cli(members: &[OrgMember]) -> bool { + members.iter().any(|member| { + parse_cli_agent_org_reference(&member.agent_id).is_some() + || members_contain_cli(&member.children) + }) + } + + let snapshot: OrgDefinition = serde_json::from_str(snapshot_json) + .map_err(|err| format!("invalid Agent Org launch snapshot: {err}"))?; + Ok(parse_cli_agent_org_reference(&snapshot.agent_id).is_some() + || members_contain_cli(&snapshot.children)) +} + +fn parse_session_status(session_id: &str, raw: &str) -> Result { + SessionStatus::parse(raw).ok_or_else(|| { + format!("Refusing to delete Agent Org: session {session_id} has unknown status {raw:?}") + }) +} + +fn validate_descendant_shape(conn: &Connection, root_session_id: &str) -> Result<(), String> { + let diagnostic = conn + .query_row( + "WITH RECURSIVE descendants(session_id, depth, path, cycle) AS ( + SELECT session_id, 0, '/' || hex(session_id) || '/', 0 + FROM agent_sessions + WHERE session_id=?1 + UNION ALL + SELECT child.session_id, + parent.depth + 1, + parent.path || hex(child.session_id) || '/', + instr(parent.path, '/' || hex(child.session_id) || '/') > 0 + FROM agent_sessions child + JOIN descendants parent ON child.parent_session_id=parent.session_id + WHERE parent.cycle=0 AND parent.depth < ?2 + LIMIT ?2 + 1 + ) + SELECT descendant.session_id, + descendant.depth, + descendant.cycle, + ( + SELECT nested.id + FROM agent_org_runs nested + WHERE nested.root_session_id=descendant.session_id + AND descendant.session_id<>?1 + ORDER BY nested.id + LIMIT 1 + ), + EXISTS( + SELECT 1 FROM agent_sessions child + WHERE child.parent_session_id=descendant.session_id + ), + (SELECT COUNT(*) FROM descendants) > ?2 + FROM descendants descendant + WHERE descendant.cycle=1 + OR descendant.depth>=?2 + OR EXISTS( + SELECT 1 FROM agent_org_runs nested + WHERE nested.root_session_id=descendant.session_id + AND descendant.session_id<>?1 + ) + OR (SELECT COUNT(*) FROM descendants) > ?2 + ORDER BY descendant.depth, descendant.session_id + LIMIT 1", + params![root_session_id, MAX_AGENT_ORG_DELETE_SESSIONS as i64], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, bool>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, bool>(4)?, + row.get::<_, bool>(5)?, + )) + }, + ) + .optional() + .map_err(|err| err.to_string())?; + if let Some((session_id, depth, cycle, nested_run_id, has_children, overflow)) = diagnostic { + if overflow { + return Err(root_refusal( + root_session_id, + "descendant diagnostics exceed the bounded limit", + )); + } + if cycle { + return Err(root_refusal( + root_session_id, + format!("descendant ancestry contains a cycle at {session_id}"), + )); + } + if let Some(nested_run_id) = nested_run_id { + return Err(root_refusal( + root_session_id, + format!( + "descendant session {session_id} owns unsupported nested run {nested_run_id}" + ), + )); + } + if depth >= MAX_AGENT_ORG_DELETE_SESSIONS as i64 && has_children { + return Err(root_refusal( + root_session_id, + "descendant diagnostics exceed the bounded limit", + )); + } + } + Ok(()) +} + +fn reject_unmapped_rust_workers(conn: &Connection, root_session_id: &str) -> Result<(), String> { + let unmapped_worker: Option = conn + .query_row( + "WITH RECURSIVE descendants(session_id, depth) AS ( + SELECT session_id, 1 + FROM agent_sessions + WHERE parent_session_id=?1 + UNION ALL + SELECT child.session_id, parent.depth + 1 + FROM agent_sessions child + JOIN descendants parent ON child.parent_session_id=parent.session_id + WHERE parent.depth < ?3 + LIMIT ?3 + 1 + ) + SELECT child.session_id + FROM descendants descendant + JOIN agent_sessions child ON child.session_id=descendant.session_id + WHERE ( + child.session_type=?2 + OR ( + child.org_member_id IS NOT NULL + AND child.org_member_id<>?4 + AND child.agent_definition_id IS NOT NULL + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM agent_org_run_sessions mapping + JOIN agent_org_runs run ON run.id=mapping.org_run_id + WHERE mapping.session_id=child.session_id + AND mapping.role='worker' + AND run.root_session_id=?1 + ) + ORDER BY child.session_id + LIMIT 1", + params![ + root_session_id, + session_type::ORG_MEMBER, + MAX_AGENT_ORG_DELETE_SESSIONS as i64, + COORDINATOR_MEMBER_ID, + ], + |row| row.get(0), + ) + .optional() + .map_err(|err| err.to_string())?; + if let Some(session_id) = unmapped_worker { + return Err(root_refusal( + root_session_id, + format!("Rust Worker session {session_id} has no exact Run ownership"), + )); + } + Ok(()) +} + +fn reject_historical_cli_descendants( + conn: &Connection, + root_session_id: &str, +) -> Result<(), String> { + let cli_session_id = conn + .query_row( + "SELECT cli.session_id + FROM code_sessions cli + WHERE cli.parent_session_id=?1 + OR EXISTS ( + SELECT 1 + FROM agent_org_run_sessions mapping + JOIN agent_org_runs run ON run.id=mapping.org_run_id + WHERE run.root_session_id=?1 + AND mapping.role='worker' + AND mapping.session_id=cli.parent_session_id + ) + ORDER BY cli.session_id + LIMIT 1", + [root_session_id], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|err| err.to_string())?; + if let Some(cli_session_id) = cli_session_id { + return Err(root_refusal( + root_session_id, + format!("unsupported historical CLI session {cli_session_id} is attached"), + )); + } + Ok(()) +} + +pub(super) fn agent_org_delete_topology_matches( + expected: &AgentOrgSessionDeletePlan, + current: &AgentOrgSessionDeletePlan, +) -> bool { + expected.root_session_id == current.root_session_id + && expected.runs.len() == current.runs.len() + && expected + .runs + .iter() + .zip(¤t.runs) + .all(|(left, right)| { + left.run_id == right.run_id && left.worker_session_ids == right.worker_session_ids + }) + && expected.sessions.len() == current.sessions.len() + && expected + .sessions + .iter() + .zip(¤t.sessions) + .all(|(left, right)| { + left.session_id == right.session_id + && left.parent_session_id == right.parent_session_id + && left.owning_run_id == right.owning_run_id + }) +} + +pub(super) fn validate_agent_org_delete_ready( + plan: &AgentOrgSessionDeletePlan, + safe_inflight_session_ids: &HashSet, +) -> Result<(), String> { + for run in &plan.runs { + if !run.status.is_terminal() { + return Err(format!( + "Refusing to delete Agent Org run {}: run status is {}", + run.run_id, + run.status.as_str() + )); + } + } + for node in &plan.sessions { + let allowed = node.status == SessionStatus::Idle + || node.status.is_terminal() + || matches!(node.status, SessionStatus::Pending | SessionStatus::Paused) + || (node.status.is_in_flight() && safe_inflight_session_ids.contains(&node.session_id)); + if !allowed { + return Err(format!( + "Refusing to delete Agent Org root {}: session {} status is {}", + plan.root_session_id, + node.session_id, + node.status.as_str() + )); + } + } + Ok(()) +} + +async fn agent_org_runtime_sessions( + state: &AgentAppState, + plan: &AgentOrgSessionDeletePlan, +) -> Vec<(String, Arc)> { + let sessions = state.sessions.lock().await; + plan.sessions + .iter() + .filter_map(|node| { + sessions + .get(&node.session_id) + .cloned() + .map(|session| (node.session_id.clone(), session)) + }) + .collect() +} + +async fn agent_org_runtime_blockers( + plan: &AgentOrgSessionDeletePlan, + runtime_sessions: &[(String, Arc)], +) -> Vec { + let mut blockers = Vec::new(); + for node in &plan.sessions { + if agent_org_submission_in_progress(&node.session_id) { + blockers.push(format!("{}(submission_in_progress=true)", node.session_id)); + } + } + for (session_id, session) in runtime_sessions { + let scheduler_processing = session.scheduler.is_processing(); + let pending_count = session.scheduler.pending_count(); + let active_turn = session.active_turn.lock().await.is_some(); + if active_turn || scheduler_processing || pending_count > 0 { + blockers.push(format!( + "{session_id}(active_turn={active_turn},scheduler_processing={scheduler_processing},pending={pending_count})" + )); + } + } + blockers +} + +fn blocker_summary(blockers: &[String]) -> String { + const MAX_SAMPLES: usize = 8; + let mut summary = blockers + .iter() + .take(MAX_SAMPLES) + .cloned() + .collect::>() + .join(", "); + if blockers.len() > MAX_SAMPLES { + summary.push_str(&format!(", … {} more", blockers.len() - MAX_SAMPLES)); + } + summary +} + +async fn stop_agent_org_runtime_sessions( + state: &AgentAppState, + plan: &AgentOrgSessionDeletePlan, +) -> Result, String> { + stop_agent_org_runtime_sessions_with_timeout(state, plan, AGENT_ORG_DELETE_STOP_TIMEOUT).await +} + +pub(super) async fn stop_agent_org_runtime_sessions_with_timeout( + state: &AgentAppState, + plan: &AgentOrgSessionDeletePlan, + timeout: Duration, +) -> Result, String> { + let deadline = tokio::time::Instant::now() + timeout; + let mut delete_cancelled_runtimes = HashMap::>::new(); + loop { + // A stale submission may have crossed its pre-init check before the + // fence. Re-snapshot on every pass so a late runtime is cancelled and + // observed rather than escaping the initial registry snapshot. + let runtime_sessions = agent_org_runtime_sessions(state, plan).await; + for (session_id, session) in &runtime_sessions { + // Resume may have committed immediately before the fence and can + // still clear the in-memory flag afterwards. A true flag alone is + // not enough: OrgPause uses the same flag without discarding + // queued work. Apply the delete reason once per runtime instance, + // and re-apply it if that instance later clears the flag. + let needs_delete_cancel = + delete_cancelled_runtimes + .get(session_id) + .is_none_or(|previous| { + !Arc::ptr_eq(previous, session) + || !session.cancel_flag.load(Ordering::SeqCst) + }); + if needs_delete_cancel { + session + .cancel_active_turn(CancelReason::AgentOrgDelete) + .await; + delete_cancelled_runtimes.insert(session_id.clone(), Arc::clone(session)); + } + } + let blockers = agent_org_runtime_blockers(plan, &runtime_sessions).await; + if blockers.is_empty() { + // Once the durable fence exists, the guarded create/wake/recovery + // paths cannot register a replacement runtime. Rows with no + // in-memory runtime are therefore safe after a process restart. + return Ok(plan.session_ids()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "Timed out stopping Agent Org root {} before deletion: {}", + plan.root_session_id, + blocker_summary(&blockers) + )); + } + tokio::time::sleep(AGENT_ORG_DELETE_STOP_POLL_INTERVAL).await; + } +} + +async fn ensure_agent_org_runtime_sessions_idle( + state: &AgentAppState, + plan: &AgentOrgSessionDeletePlan, +) -> Result<(), String> { + let runtime_sessions = agent_org_runtime_sessions(state, plan).await; + let blockers = agent_org_runtime_blockers(plan, &runtime_sessions).await; + if blockers.is_empty() { + Ok(()) + } else { + Err(format!( + "Refusing to delete Agent Org root {}: active Rust runtime sessions: {}", + plan.root_session_id, + blocker_summary(&blockers) + )) + } +} + +#[cfg(test)] +pub(super) fn delete_agent_org_session_hierarchy( + expected_plan: &AgentOrgSessionDeletePlan, + safe_inflight_session_ids: &HashSet, +) -> Result { + let committed_delete = + commit_agent_org_session_hierarchy(expected_plan, safe_inflight_session_ids)?; + finish_agent_org_post_commit_resources( + committed_delete.run_cleanup, + committed_delete.session_cleanup, + ); + Ok(committed_delete.receipt) +} + +fn commit_agent_org_session_hierarchy( + expected_plan: &AgentOrgSessionDeletePlan, + safe_inflight_session_ids: &HashSet, +) -> Result { + preflight_agent_org_delete_resources(expected_plan)?; + let (run_outcomes, deleted_session_ids, post_commit_cleanup, committed_delete) = + with_sessions_writer(|| { + let mut conn = get_connection().map_err(|err| err.to_string())?; + let tx = conn + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|err| err.to_string())?; + let current_plan = match load_agent_org_session_delete_plan( + &tx, + &expected_plan.root_session_id, + )? { + Some(plan) => plan, + None => { + let receipt = completed_agent_org_delete_receipt(&tx, expected_plan)? + .ok_or_else(|| { + format!( + "Refusing to delete Agent Org root {}: ownership changed before deletion", + expected_plan.root_session_id + ) + })?; + tx.commit().map_err(|err| err.to_string())?; + return Ok::<_, String>(( + Vec::new(), + receipt.deleted_session_ids, + Vec::new(), + false, + )); + } + }; + if !agent_org_delete_topology_matches(expected_plan, ¤t_plan) { + return Err(format!( + "Refusing to delete Agent Org root {}: ownership changed before deletion", + expected_plan.root_session_id + )); + } + validate_agent_org_delete_ready(¤t_plan, safe_inflight_session_ids)?; + if let Some(node) = current_plan + .sessions + .iter() + .find(|node| agent_org_submission_in_progress(&node.session_id)) + { + return Err(format!( + "Refusing to delete Agent Org root {}: session {} submission is still initializing", + current_plan.root_session_id, node.session_id + )); + } + + let mut post_commit_cleanup = Vec::with_capacity(current_plan.sessions.len()); + for node in ¤t_plan.sessions { + let (workspace_path, base_branch): (Option, Option) = tx + .query_row( + "SELECT workspace_path, base_branch + FROM agent_sessions + WHERE session_id=?1", + [&node.session_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|err| { + format!("load cleanup context for {}: {err}", node.session_id) + })?; + post_commit_cleanup.push(AgentOrgSessionPostCommitCleanup { + session_id: node.session_id.clone(), + workspace_path: workspace_path.map(PathBuf::from), + managed_worktree: base_branch.is_some(), + }); + for table in SESSION_STATE_TABLES { + tx.execute( + &format!("DELETE FROM {table} WHERE session_id=?1"), + [&node.session_id], + ) + .map_err(|err| { + format!("delete {table} for session {}: {err}", node.session_id) + })?; + } + tx.execute( + "DELETE FROM session_turn_intents WHERE session_id=?1", + [&node.session_id], + ) + .map_err(|err| { + format!("delete Turn Intents for session {}: {err}", node.session_id) + })?; + session_persistence::delete_session_with_connection(&tx, &node.session_id) + .map_err(|err| format!("delete session {}: {err}", node.session_id))?; + } + + let mut outcomes = Vec::with_capacity(current_plan.runs.len()); + for run in ¤t_plan.runs { + let outcome = AgentOrgRunStore::delete_by_id_with_connection(&tx, &run.run_id)?; + if !outcome.deleted() { + return Err(format!( + "Refusing to commit Agent Org run {} deletion: run row disappeared during deletion", + run.run_id + )); + } + outcomes.push((run.run_id.clone(), outcome)); + } + if !remove_conversation_delete_fence_with_connection( + &tx, + ¤t_plan.root_session_id, + )? { + return Err(format!( + "Refusing to commit Agent Org root {} deletion: deletion fence disappeared", + current_plan.root_session_id + )); + } + ensure_agent_org_sessions_absent(&tx, ¤t_plan)?; + let deleted_session_ids = current_plan + .sessions + .iter() + .map(|node| node.session_id.clone()) + .collect::>(); + tx.commit().map_err(|err| err.to_string())?; + Ok::<_, String>((outcomes, deleted_session_ids, post_commit_cleanup, true)) + })?; + + Ok(AgentOrgCommittedDelete { + receipt: DeleteSessionReceipt { + deleted_session_ids, + }, + run_cleanup: if committed_delete { + run_outcomes + } else { + Vec::new() + }, + session_cleanup: post_commit_cleanup, + }) +} + +fn finish_agent_org_post_commit_resources( + run_cleanup: Vec<(String, AgentOrgRunDeleteOutcome)>, + session_cleanup: Vec, +) { + for (run_id, outcome) in run_cleanup { + AgentOrgRunStore::finish_delete(&run_id, outcome); + } + for cleanup in session_cleanup { + session_persistence::finish_session_delete(&cleanup.session_id); + cleanup_agent_org_scratchpad(&cleanup.session_id); + if cleanup.managed_worktree { + if let Some(workspace_path) = cleanup.workspace_path { + if let Err(error) = git::worktree::remove_session_worktree( + &workspace_path, + &cleanup.session_id, + true, + ) { + tracing::warn!( + session_id = %cleanup.session_id, + error = %error, + "Agent Org Session committed deleted, but managed worktree cleanup failed" + ); + } + } + } + } +} + +fn cleanup_agent_org_scratchpad(session_id: &str) { + app_paths::cleanup_scratchpad_by_session_id(session_id); + let root = app_paths::orgii_temp_root(); + if !root.exists() { + return; + } + let entries = match std::fs::read_dir(&root) { + Ok(entries) => entries, + Err(error) => { + tracing::warn!(session_id, %error, "Agent Org scratchpad cleanup could not be verified"); + return; + } + }; + for entry in entries.flatten() { + let is_dir = entry.file_type().is_ok_and(|file_type| file_type.is_dir()); + let session_dir = entry.path().join(session_id); + if is_dir && session_dir.exists() { + tracing::warn!( + session_id, + path = %session_dir.display(), + "Agent Org Session committed deleted, but scratchpad cleanup failed" + ); + } + } +} + +fn ensure_agent_org_sessions_absent( + conn: &Connection, + plan: &AgentOrgSessionDeletePlan, +) -> Result<(), String> { + for node in &plan.sessions { + let exists = conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM agent_sessions WHERE session_id=?1)", + [&node.session_id], + |row| row.get::<_, bool>(0), + ) + .map_err(|err| err.to_string())?; + if exists { + return Err(format!( + "Refusing to commit Agent Org root {} deletion: residual session {}", + plan.root_session_id, node.session_id + )); + } + } + Ok(()) +} + +impl AgentOrgSessionDeletePlan { + fn session_ids(&self) -> HashSet { + self.sessions + .iter() + .map(|node| node.session_id.clone()) + .collect() + } +} diff --git a/src-tauri/crates/agent-core/src/state/commands/session/agent_org_delete_tests.rs b/src-tauri/crates/agent-core/src/state/commands/session/agent_org_delete_tests.rs new file mode 100644 index 000000000..b20294f18 --- /dev/null +++ b/src-tauri/crates/agent-core/src/state/commands/session/agent_org_delete_tests.rs @@ -0,0 +1,939 @@ +use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use rusqlite::{params, Connection}; + +use crate::coordination::agent_org_runs::{ + agent_org_submission_in_progress, establish_conversation_delete_fence, AgentOrgRunStatus, + AgentOrgSubmissionLease, COORDINATOR_MEMBER_ID, +}; +use crate::session::persistence::{self as session_persistence, session_type}; +use crate::session::scheduler::ExecuteFn; +use crate::session::{ScheduledKind, ScheduledMessage, SessionStatus}; +use crate::state::{AgentAppState, AgentSession}; + +use super::agent_org_delete::{ + agent_org_delete_topology_matches, delete_agent_org_session_hierarchy, + load_agent_org_session_delete_plan, stop_agent_org_runtime_sessions_with_timeout, + validate_agent_org_delete_ready, AgentOrgRunDeletePlan, AgentOrgSessionDeleteNode, + AgentOrgSessionDeletePlan, MAX_AGENT_ORG_DELETE_RUNS, MAX_AGENT_ORG_DELETE_SESSIONS, +}; + +const NOW: &str = "2026-08-02T00:00:00Z"; + +fn ensure_schemas() { + let conn = database::db::get_connection().expect("sandbox DB"); + crate::foundation::persistence::test_schema::ensure_agent_sessions_schema(&conn); + crate::foundation::persistence::session_snapshots::ensure_tables_with(&conn) + .expect("snapshot schema"); + session_persistence::init(&conn).expect("Session schema"); + crate::interaction::plan_approval::persistence::init_schema(&conn).expect("approval schema"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); + project_management::lineage::schema::init_lineage_tables(&conn).expect("lineage schema"); + crate::memory::learnings::init_learnings_table(&conn).expect("learnings schema"); + database::init_shell_replay_tables(&conn).expect("shell replay schema"); + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS events (id TEXT PRIMARY KEY,session_id TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS code_sessions (session_id TEXT PRIMARY KEY,cli_agent_type TEXT NOT NULL,status TEXT NOT NULL,parent_session_id TEXT,org_member_id TEXT,updated_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS session_turn_intents (session_id TEXT NOT NULL,turn_intent_id TEXT NOT NULL,client_message_id TEXT,org_run_id TEXT,source TEXT NOT NULL,status TEXT NOT NULL,created_at TEXT NOT NULL,updated_at TEXT NOT NULL,PRIMARY KEY(session_id,turn_intent_id)); + CREATE TABLE session_turns (session_id TEXT); CREATE TABLE session_turn_index_state (session_id TEXT); CREATE TABLE sessions (session_id TEXT); CREATE TABLE goal_loop_state (session_id TEXT); CREATE TABLE housekeeper_context_compaction (session_id TEXT);", + ) + .expect("runtime schemas"); +} + +fn seed_session( + conn: &Connection, + session_id: &str, + parent_session_id: Option<&str>, + status: SessionStatus, + kind: &str, + member_id: Option<&str>, +) { + conn.execute( + "INSERT INTO agent_sessions (session_id,name,status,created_at,updated_at,session_type,agent_definition_id,org_member_id,parent_session_id,workspace_additional_json,key_source) VALUES (?1,?1,?2,?3,?3,?4,?5,?6,?7,'{}','own_key')", + params![ + session_id, + status.as_str(), + NOW, + kind, + member_id.map(|_| "agent-worker"), + member_id, + parent_session_id + ], + ) + .expect("seed Session"); +} + +fn seed_root(conn: &Connection, session_id: &str) { + seed_session( + conn, + session_id, + None, + SessionStatus::Idle, + session_type::GENERIC, + None, + ); +} + +fn seed_plain_child(conn: &Connection, session_id: &str, parent: &str) { + seed_session( + conn, + session_id, + Some(parent), + SessionStatus::Idle, + session_type::GENERIC, + None, + ); +} + +fn seed_run(conn: &Connection, run_id: &str, root: &str, status: AgentOrgRunStatus) { + conn.execute( + "INSERT INTO agent_org_runs (id,org_id,coordinator_agent_id,root_session_id,entry_mode,status,created_at,updated_at) VALUES (?1,'org-delete-test','coordinator-agent',?2,'standalone_session',?3,?4,?4)", + params![run_id, root, status.as_str(), NOW], + ) + .expect("seed Run"); + conn.execute( + "INSERT INTO agent_org_run_sessions (org_run_id,member_id,session_id,role,created_at) VALUES (?1,?2,?3,'coordinator',?4)", + params![run_id, COORDINATOR_MEMBER_ID, root, NOW], + ) + .expect("seed Coordinator mapping"); +} + +fn seed_done_run(conn: &Connection, run_id: &str, root: &str) { + seed_run(conn, run_id, root, AgentOrgRunStatus::Completed); +} + +fn seed_worker(conn: &Connection, run_id: &str, worker: &str, root: &str, member: &str) { + seed_session( + conn, + worker, + Some(root), + SessionStatus::Completed, + session_type::ORG_MEMBER, + Some(member), + ); + conn.execute( + "INSERT INTO agent_org_run_sessions (org_run_id,member_id,session_id,role,created_at) VALUES (?1,?2,?3,'worker',?4)", + params![run_id, member, worker, NOW], + ) + .expect("seed Worker mapping"); +} + +fn seed_owned_rows(conn: &Connection, session_id: &str) { + for table in [ + "session_turns", + "session_turn_index_state", + "sessions", + "goal_loop_state", + "housekeeper_context_compaction", + ] { + conn.execute( + &format!("INSERT INTO {table} (session_id) VALUES (?1)"), + [session_id], + ) + .unwrap(); + } + conn.execute( + "INSERT INTO agent_messages (id,session_id,role,content,sequence,created_at) VALUES (?1,?2,'user','delete me',0,?3)", + params![format!("message-{session_id}"), session_id, NOW], + ) + .unwrap(); + conn.execute( + "INSERT INTO agent_todos (session_id,content) VALUES (?1,'delete me')", + [session_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO events (id,session_id) VALUES (?1,?2)", + params![format!("event-{session_id}"), session_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO session_turn_intents (session_id,turn_intent_id,source,status,created_at,updated_at) VALUES (?1,?2,'session','pending',?3,?3)", + params![session_id, format!("intent-session-{session_id}"), NOW], + ) + .unwrap(); +} + +fn seed_run_rows(conn: &Connection, run_id: &str) { + conn.execute( + "INSERT INTO agent_org_tasks (id,org_run_id,subject,status,created_at,updated_at) VALUES (?1,?2,'delete me','completed',?3,?3)", + params![format!("task-{run_id}"), run_id, NOW], + ) + .unwrap(); + conn.execute( + "INSERT INTO agent_inbox (recipient_agent_id,recipient_member_id,sender_agent_id,org_run_id,payload_kind,payload_json,created_at) VALUES ('worker-agent','worker','system',?1,'plain','{\"summary\":\"delete me\",\"text\":\"body\"}',?2)", + params![run_id, NOW], + ) + .unwrap(); + conn.execute( + "INSERT INTO session_turn_intents (session_id,turn_intent_id,org_run_id,source,status,created_at,updated_at) VALUES (?1,?2,?3,'agent_org','pending',?4,?4)", + params![format!("run-owner-{run_id}"), format!("intent-run-{run_id}"), run_id, NOW], + ) + .unwrap(); +} + +fn exists(conn: &Connection, table: &str, column: &str, value: &str) -> bool { + conn.query_row( + &format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE {column}=?1)"), + [value], + |row| row.get(0), + ) + .expect("inspect row") +} + +fn fenced_plan(root: &str) -> AgentOrgSessionDeletePlan { + establish_conversation_delete_fence(root).expect("establish durable fence"); + let conn = database::db::get_connection().unwrap(); + load_agent_org_session_delete_plan(&conn, root) + .expect("load delete plan") + .expect("Root owns Runs") +} + +#[test] +fn deletion_submission_lease_is_reference_counted() { + let session_id = format!("delete-submission-lease-{}", uuid::Uuid::new_v4()); + assert!(!agent_org_submission_in_progress(&session_id)); + let first = AgentOrgSubmissionLease::begin(&session_id); + let second = AgentOrgSubmissionLease::begin(&session_id); + drop(first); + assert!(agent_org_submission_in_progress(&session_id)); + drop(second); + assert!(!agent_org_submission_in_progress(&session_id)); +} + +#[test] +fn session_hierarchy_delete_removes_exact_multi_run_ownership() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let conn = database::db::get_connection().unwrap(); + let root = "delete-multi-root"; + seed_root(&conn, root); + seed_done_run(&conn, "delete-run-a", root); + seed_run(&conn, "delete-run-b", root, AgentOrgRunStatus::Paused); + seed_worker(&conn, "delete-run-a", "delete-worker-a", root, "worker-a"); + seed_worker(&conn, "delete-run-b", "delete-worker-b", root, "worker-b"); + conn.execute( + "UPDATE agent_sessions SET session_type=?1 + WHERE session_id='delete-worker-a'", + [session_type::GENERIC], + ) + .unwrap(); + seed_plain_child(&conn, "delete-plain-child", root); + seed_root(&conn, "delete-unrelated-root"); + seed_done_run(&conn, "delete-unrelated-run", "delete-unrelated-root"); + seed_worker( + &conn, + "delete-unrelated-run", + "delete-unrelated-worker", + "delete-unrelated-root", + "unrelated-worker", + ); + for session in [ + root, + "delete-worker-a", + "delete-worker-b", + "delete-unrelated-root", + ] { + seed_owned_rows(&conn, session); + } + for run in ["delete-run-a", "delete-run-b", "delete-unrelated-run"] { + seed_run_rows(&conn, run); + } + let scratch_workspace = tempfile::TempDir::new().unwrap(); + let scratchpad = app_paths::ensure_scratchpad(root, scratch_workspace.path()).unwrap(); + std::fs::write(scratchpad.join("delete-me.txt"), "ephemeral").unwrap(); + drop(conn); + + let plan = fenced_plan(root); + assert_eq!(plan.runs.len(), 2); + let safe = plan + .sessions + .iter() + .map(|node| node.session_id.clone()) + .collect::>(); + let receipt = delete_agent_org_session_hierarchy(&plan, &safe).expect("atomic delete"); + assert_eq!( + receipt + .deleted_session_ids + .into_iter() + .collect::>(), + HashSet::from([ + root.to_string(), + "delete-worker-a".to_string(), + "delete-worker-b".to_string(), + ]) + ); + + let conn = database::db::get_connection().unwrap(); + for session in [root, "delete-worker-a", "delete-worker-b"] { + assert!(!exists(&conn, "agent_sessions", "session_id", session)); + assert!(!exists( + &conn, + "session_turn_intents", + "session_id", + session + )); + for table in [ + "session_turns", + "session_turn_index_state", + "sessions", + "goal_loop_state", + "housekeeper_context_compaction", + ] { + assert!(!exists(&conn, table, "session_id", session)); + } + } + for run in ["delete-run-a", "delete-run-b"] { + assert!(!exists(&conn, "agent_org_runs", "id", run)); + assert!(!exists(&conn, "agent_org_tasks", "org_run_id", run)); + assert!(!exists(&conn, "agent_inbox", "org_run_id", run)); + assert!(!exists(&conn, "session_turn_intents", "org_run_id", run)); + } + assert!(!exists( + &conn, + "agent_org_conversation_delete_fences", + "root_session_id", + root, + )); + for session in [ + "delete-plain-child", + "delete-unrelated-root", + "delete-unrelated-worker", + ] { + assert!(exists(&conn, "agent_sessions", "session_id", session)); + } + assert!(exists( + &conn, + "agent_org_runs", + "id", + "delete-unrelated-run" + )); + assert!(!scratchpad.exists()); +} + +#[test] +fn session_hierarchy_delete_rolls_back_then_retries_idempotently() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let conn = database::db::get_connection().unwrap(); + let root = "delete-rollback-root"; + seed_root(&conn, root); + for suffix in ["a", "b"] { + let run = format!("delete-rollback-run-{suffix}"); + let worker = format!("delete-rollback-worker-{suffix}"); + seed_done_run(&conn, &run, root); + seed_worker(&conn, &run, &worker, root, &format!("worker-{suffix}")); + seed_owned_rows(&conn, &worker); + seed_run_rows(&conn, &run); + } + seed_owned_rows(&conn, root); + drop(conn); + let plan = fenced_plan(root); + let conn = database::db::get_connection().unwrap(); + conn.execute_batch( + "CREATE TRIGGER abort_multi_run_root_delete + BEFORE DELETE ON agent_sessions + WHEN OLD.session_id='delete-rollback-root' + BEGIN SELECT RAISE(ABORT, 'injected multi-run delete failure'); END;", + ) + .unwrap(); + drop(conn); + + let error = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) + .expect_err("trigger aborts the entire transaction"); + assert!(error.contains("injected multi-run delete failure")); + let conn = database::db::get_connection().unwrap(); + for session in [root, "delete-rollback-worker-a", "delete-rollback-worker-b"] { + assert!(exists(&conn, "agent_sessions", "session_id", session)); + assert!(exists(&conn, "agent_messages", "session_id", session)); + assert!(exists(&conn, "session_turns", "session_id", session)); + } + for run in ["delete-rollback-run-a", "delete-rollback-run-b"] { + assert!(exists(&conn, "agent_org_runs", "id", run)); + assert!(exists(&conn, "agent_org_tasks", "org_run_id", run)); + } + assert!(exists( + &conn, + "agent_org_conversation_delete_fences", + "root_session_id", + root, + )); + conn.execute_batch("DROP TRIGGER abort_multi_run_root_delete") + .unwrap(); + drop(conn); + + let first = delete_agent_org_session_hierarchy(&plan, &HashSet::new()).unwrap(); + let repeated = delete_agent_org_session_hierarchy(&plan, &HashSet::new()).unwrap(); + assert_eq!(repeated.deleted_session_ids, first.deleted_session_ids); +} + +#[test] +fn session_hierarchy_delete_fails_closed_on_invalid_ownership() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let conn = database::db::get_connection().unwrap(); + + seed_root(&conn, "delete-unmapped-root"); + seed_done_run(&conn, "delete-unmapped-run", "delete-unmapped-root"); + seed_plain_child(&conn, "delete-unmapped-parent", "delete-unmapped-root"); + seed_session( + &conn, + "delete-unmapped-worker", + Some("delete-unmapped-parent"), + SessionStatus::Completed, + session_type::GENERIC, + Some("unmapped-worker"), + ); + conn.execute( + "UPDATE agent_sessions SET agent_definition_id='agent-worker' + WHERE session_id='delete-unmapped-worker'", + [], + ) + .unwrap(); + let error = load_agent_org_session_delete_plan(&conn, "delete-unmapped-root").unwrap_err(); + assert!(error.contains("no exact Run ownership")); + + seed_root(&conn, "delete-missing-map-root"); + conn.execute( + "INSERT INTO agent_org_runs (id,org_id,coordinator_agent_id,root_session_id,entry_mode,status,created_at,updated_at) VALUES ('delete-missing-map-run','org','agent','delete-missing-map-root','standalone_session','completed',?1,?1)", + [NOW], + ) + .unwrap(); + let error = load_agent_org_session_delete_plan(&conn, "delete-missing-map-root").unwrap_err(); + assert!(error.contains("Coordinator mapping is missing")); + + seed_root(&conn, "delete-nested-root"); + seed_done_run(&conn, "delete-nested-outer-run", "delete-nested-root"); + seed_plain_child(&conn, "delete-inner-root", "delete-nested-root"); + seed_done_run(&conn, "delete-nested-inner-run", "delete-inner-root"); + let error = load_agent_org_session_delete_plan(&conn, "delete-nested-root").unwrap_err(); + assert!(error.contains("unsupported nested run")); + + seed_root(&conn, "delete-cli-root"); + seed_done_run(&conn, "delete-cli-run", "delete-cli-root"); + conn.execute( + "INSERT INTO code_sessions (session_id,cli_agent_type,status,parent_session_id,updated_at) VALUES ('delete-cli-child','codex','completed','delete-cli-root',?1)", + [NOW], + ) + .unwrap(); + let error = load_agent_org_session_delete_plan(&conn, "delete-cli-root").unwrap_err(); + assert!(error.contains("historical CLI")); + + seed_root(&conn, "delete-worker-cli-root"); + seed_done_run(&conn, "delete-worker-cli-run", "delete-worker-cli-root"); + seed_worker( + &conn, + "delete-worker-cli-run", + "delete-worker-cli-worker", + "delete-worker-cli-root", + "worker", + ); + conn.execute( + "INSERT INTO code_sessions (session_id,cli_agent_type,status,parent_session_id,updated_at) VALUES ('delete-worker-cli-child','codex','completed','delete-worker-cli-worker',?1)", + [NOW], + ) + .unwrap(); + let error = load_agent_org_session_delete_plan(&conn, "delete-worker-cli-root").unwrap_err(); + assert!(error.contains("historical CLI")); + + seed_root(&conn, "delete-cli-snapshot-root"); + seed_done_run(&conn, "delete-cli-snapshot-run", "delete-cli-snapshot-root"); + conn.execute( + "UPDATE agent_org_runs SET org_snapshot_json=?1 WHERE id='delete-cli-snapshot-run'", + [r#"{"id":"legacy-cli-org","name":"Legacy CLI Org","role":"lead","agentId":"builtin:general","children":[{"id":"cli-worker","name":"CLI Worker","role":"worker","agentId":"cli:claude_code","children":[]}]}"#], + ) + .unwrap(); + let error = load_agent_org_session_delete_plan(&conn, "delete-cli-snapshot-root").unwrap_err(); + assert!(error.contains("CLI members are unsupported")); + + for root in [ + "delete-unmapped-root", + "delete-missing-map-root", + "delete-nested-root", + "delete-cli-root", + "delete-worker-cli-root", + "delete-cli-snapshot-root", + ] { + assert!(!exists( + &conn, + "agent_org_conversation_delete_fences", + "root_session_id", + root, + )); + } +} + +#[test] +fn fenced_root_without_runs_never_falls_back_to_ordinary_delete() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let conn = database::db::get_connection().unwrap(); + let root = "delete-orphan-fence-root"; + seed_root(&conn, root); + seed_done_run(&conn, "delete-orphan-fence-run", root); + drop(conn); + establish_conversation_delete_fence(root).unwrap(); + let conn = database::db::get_connection().unwrap(); + conn.execute( + "DELETE FROM agent_org_run_sessions WHERE org_run_id='delete-orphan-fence-run'", + [], + ) + .unwrap(); + conn.execute( + "DELETE FROM agent_org_runs WHERE id='delete-orphan-fence-run'", + [], + ) + .unwrap(); + + let error = load_agent_org_session_delete_plan(&conn, root).unwrap_err(); + assert!(error.contains("conversation_deleting")); + assert!(exists(&conn, "agent_sessions", "session_id", root)); + assert!(exists( + &conn, + "agent_org_conversation_delete_fences", + "root_session_id", + root, + )); +} + +#[test] +fn session_hierarchy_delete_enforces_run_limit_and_rechecks_topology() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let mut conn = database::db::get_connection().unwrap(); + let root = "delete-run-limit-root"; + seed_root(&conn, root); + let tx = conn.transaction().unwrap(); + for index in 0..MAX_AGENT_ORG_DELETE_RUNS { + seed_done_run(&tx, &format!("delete-limit-run-{index:04}"), root); + } + tx.commit().unwrap(); + let plan = load_agent_org_session_delete_plan(&conn, root) + .unwrap() + .unwrap(); + assert_eq!(plan.runs.len(), MAX_AGENT_ORG_DELETE_RUNS); + seed_done_run(&conn, "delete-limit-run-overflow", root); + let error = load_agent_org_session_delete_plan(&conn, root).unwrap_err(); + assert!(error.contains("Run ownership exceeds")); + + let topology_root = "delete-topology-root"; + seed_root(&conn, topology_root); + seed_done_run(&conn, "delete-topology-run-a", topology_root); + seed_worker( + &conn, + "delete-topology-run-a", + "delete-topology-worker-a", + topology_root, + "worker-a", + ); + drop(conn); + let expected = fenced_plan(topology_root); + let conn = database::db::get_connection().unwrap(); + seed_done_run(&conn, "delete-topology-run-b", topology_root); + let current = load_agent_org_session_delete_plan(&conn, topology_root) + .unwrap() + .unwrap(); + assert!(!agent_org_delete_topology_matches(&expected, ¤t)); + drop(conn); + let error = delete_agent_org_session_hierarchy(&expected, &HashSet::new()).unwrap_err(); + assert!(error.contains("ownership changed before deletion")); +} + +#[test] +fn session_hierarchy_delete_enforces_unique_session_limit_plus_one() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let mut conn = database::db::get_connection().unwrap(); + let root = "delete-session-limit-root"; + let run = "delete-session-limit-run"; + seed_root(&conn, root); + seed_done_run(&conn, run, root); + let tx = conn.transaction().unwrap(); + for index in 0..(MAX_AGENT_ORG_DELETE_SESSIONS - 1) { + seed_worker( + &tx, + run, + &format!("delete-session-limit-worker-{index:04}"), + root, + &format!("worker-{index:04}"), + ); + } + tx.commit().unwrap(); + let plan = load_agent_org_session_delete_plan(&conn, root) + .unwrap() + .unwrap(); + assert_eq!(plan.sessions.len(), MAX_AGENT_ORG_DELETE_SESSIONS); + + seed_worker( + &conn, + run, + "delete-session-limit-overflow", + root, + "worker-overflow", + ); + let error = load_agent_org_session_delete_plan(&conn, root).unwrap_err(); + assert!(error.contains("exceed")); +} + +#[test] +fn concurrent_hierarchy_deletes_are_complete_and_idempotent() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let conn = database::db::get_connection().unwrap(); + let root = "delete-concurrent-root"; + seed_root(&conn, root); + seed_done_run(&conn, "delete-concurrent-run", root); + seed_worker( + &conn, + "delete-concurrent-run", + "delete-concurrent-worker", + root, + "worker", + ); + drop(conn); + let plan = fenced_plan(root); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let handles = (0..2) + .map(|_| { + let plan = plan.clone(); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + delete_agent_org_session_hierarchy(&plan, &HashSet::new()) + }) + }) + .collect::>(); + let receipts = handles + .into_iter() + .map(|handle| handle.join().unwrap().unwrap()) + .collect::>(); + assert_eq!( + receipts[0].deleted_session_ids, + receipts[1].deleted_session_ids + ); + assert_eq!(receipts[0].deleted_session_ids.len(), 2); +} + +#[test] +fn direct_worker_delete_preserves_root_and_run() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let conn = database::db::get_connection().unwrap(); + let root = "delete-direct-worker-root"; + let run = "delete-direct-worker-run"; + let worker = "delete-direct-worker"; + seed_root(&conn, root); + seed_done_run(&conn, run, root); + seed_worker(&conn, run, worker, root, "worker"); + assert!(load_agent_org_session_delete_plan(&conn, worker) + .unwrap() + .is_none()); + drop(conn); + establish_conversation_delete_fence(root).unwrap(); + let conn = database::db::get_connection().unwrap(); + let error = load_agent_org_session_delete_plan(&conn, worker).unwrap_err(); + assert!(error.contains("conversation_deleting")); + conn.execute( + "DELETE FROM agent_org_conversation_delete_fences WHERE root_session_id=?1", + [root], + ) + .unwrap(); + drop(conn); + session_persistence::delete_session(worker).unwrap(); + let conn = database::db::get_connection().unwrap(); + assert!(exists(&conn, "agent_sessions", "session_id", root)); + assert!(exists(&conn, "agent_org_runs", "id", run)); +} + +fn runtime_plan(root: &str, worker: Option<&str>) -> AgentOrgSessionDeletePlan { + let mut sessions = vec![AgentOrgSessionDeleteNode { + session_id: root.to_string(), + parent_session_id: None, + status: SessionStatus::Running, + owning_run_id: None, + }]; + let mut workers = Vec::new(); + if let Some(worker) = worker { + workers.push(worker.to_string()); + sessions.push(AgentOrgSessionDeleteNode { + session_id: worker.to_string(), + parent_session_id: Some(root.to_string()), + status: SessionStatus::Pending, + owning_run_id: Some("delete-runtime-run".to_string()), + }); + } + AgentOrgSessionDeletePlan { + root_session_id: root.to_string(), + runs: vec![AgentOrgRunDeletePlan { + run_id: "delete-runtime-run".to_string(), + status: AgentOrgRunStatus::Cancelled, + worker_session_ids: workers, + }], + sessions, + } +} + +fn scheduled_message(kind: ScheduledKind, id: &str, execute: ExecuteFn) -> ScheduledMessage { + ScheduledMessage { + kind, + message_id: id.to_string(), + generation: 0, + client_message_id: None, + turn_intent_id: format!("{id}-intent"), + org_run_id: Some("delete-runtime-run".to_string()), + content: String::new(), + execute, + } +} + +#[tokio::test] +async fn session_hierarchy_delete_quiesces_runtime_and_times_out_safely() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + + let initializing_state = AgentAppState::new(); + let submission_guard = AgentOrgSubmissionLease::begin("delete-initializing-root"); + let initializing_plan = runtime_plan("delete-initializing-root", None); + let error = stop_agent_org_runtime_sessions_with_timeout( + &initializing_state, + &initializing_plan, + Duration::from_millis(50), + ) + .await + .unwrap_err(); + assert!(error.contains("submission_in_progress")); + drop(submission_guard); + stop_agent_org_runtime_sessions_with_timeout( + &initializing_state, + &initializing_plan, + Duration::from_millis(50), + ) + .await + .unwrap(); + + let paused_state = AgentAppState::new(); + let paused = Arc::new(AgentSession::new( + "delete-paused-runtime".to_string(), + crate::definitions::AgentDefinition::default(), + )); + paused + .steering_queue + .lock() + .await + .push(crate::turn_executor::SteeringInjection { + content: "must be discarded".to_string(), + turn_intent_id: "delete-paused-steering".to_string(), + }); + paused + .cancel_active_turn(crate::state::control_flow::CancelReason::OrgPause) + .await; + assert_eq!(paused.steering_queue.lock().await.len(), 1); + paused_state + .sessions + .lock() + .await + .insert(paused.id.clone(), Arc::clone(&paused)); + stop_agent_org_runtime_sessions_with_timeout( + &paused_state, + &runtime_plan("delete-paused-runtime", None), + Duration::from_secs(1), + ) + .await + .unwrap(); + assert!(paused.steering_queue.lock().await.is_empty()); + + let state = AgentAppState::new(); + let runtime = Arc::new(AgentSession::new( + "delete-runtime-root".to_string(), + crate::definitions::AgentDefinition::default(), + )); + let started = Arc::new(tokio::sync::Notify::new()); + let started_for_job = Arc::clone(&started); + let runtime_for_job = Arc::clone(&runtime); + runtime + .scheduler + .enqueue(scheduled_message( + ScheduledKind::Turn, + "delete-runtime-active", + Box::new(move || { + let runtime = Arc::clone(&runtime_for_job); + let started = Arc::clone(&started_for_job); + Box::pin(async move { + runtime.begin_turn("running".to_string()).await; + started.notify_one(); + while !runtime.cancel_flag.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + runtime + .end_turn( + crate::session::DialogTurnState::Cancelled, + crate::session::TurnStats::default(), + ) + .await; + Err("cancelled".to_string()) + }) + }), + )) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), started.notified()) + .await + .unwrap(); + let pending_executed = Arc::new(AtomicBool::new(false)); + let pending_for_job = Arc::clone(&pending_executed); + runtime + .scheduler + .enqueue(scheduled_message( + ScheduledKind::Turn, + "delete-runtime-pending", + Box::new(move || { + let executed = Arc::clone(&pending_for_job); + Box::pin(async move { + executed.store(true, Ordering::SeqCst); + Ok(String::new()) + }) + }), + )) + .await + .unwrap(); + state + .sessions + .lock() + .await + .insert(runtime.id.clone(), Arc::clone(&runtime)); + let plan = runtime_plan("delete-runtime-root", Some("delete-runtime-pending-worker")); + let safe = stop_agent_org_runtime_sessions_with_timeout(&state, &plan, Duration::from_secs(1)) + .await + .unwrap(); + assert!(safe.contains("delete-runtime-pending-worker")); + assert_eq!(runtime.scheduler.pending_count(), 0); + assert!(!pending_executed.load(Ordering::SeqCst)); + validate_agent_org_delete_ready(&plan, &safe).unwrap(); + + let timeout_state = AgentAppState::new(); + let stuck = Arc::new(AgentSession::new( + "delete-runtime-stuck".to_string(), + crate::definitions::AgentDefinition::default(), + )); + let release = Arc::new(tokio::sync::Notify::new()); + let release_for_job = Arc::clone(&release); + stuck + .scheduler + .enqueue(scheduled_message( + ScheduledKind::Maintenance, + "delete-runtime-stuck-job", + Box::new(move || { + let release = Arc::clone(&release_for_job); + Box::pin(async move { + release.notified().await; + Ok(String::new()) + }) + }), + )) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !stuck.scheduler.is_processing() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + timeout_state + .sessions + .lock() + .await + .insert(stuck.id.clone(), Arc::clone(&stuck)); + let error = stop_agent_org_runtime_sessions_with_timeout( + &timeout_state, + &runtime_plan("delete-runtime-stuck", None), + Duration::from_millis(50), + ) + .await + .unwrap_err(); + assert!(error.contains("Timed out stopping")); + assert!(timeout_state + .get_session("delete-runtime-stuck") + .await + .is_some()); + release.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + while stuck.scheduler.is_processing() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn session_hierarchy_delete_reasserts_cancel_after_resume_race() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_schemas(); + let state = Arc::new(AgentAppState::new()); + let runtime = Arc::new(AgentSession::new( + "delete-resume-race-root".to_string(), + crate::definitions::AgentDefinition::default(), + )); + let release = Arc::new(tokio::sync::Notify::new()); + let release_for_job = Arc::clone(&release); + runtime + .scheduler + .enqueue(scheduled_message( + ScheduledKind::Maintenance, + "delete-resume-race-job", + Box::new(move || { + let release = Arc::clone(&release_for_job); + Box::pin(async move { + release.notified().await; + Ok(String::new()) + }) + }), + )) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !runtime.scheduler.is_processing() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + state + .sessions + .lock() + .await + .insert(runtime.id.clone(), Arc::clone(&runtime)); + let state_for_delete = Arc::clone(&state); + let delete_task = tokio::spawn(async move { + stop_agent_org_runtime_sessions_with_timeout( + &state_for_delete, + &runtime_plan("delete-resume-race-root", None), + Duration::from_secs(1), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while !runtime.cancel_flag.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + // Simulate the post-resume flag clear racing immediately behind the + // durable fence. The deletion loop must observe and re-assert its reason. + runtime.cancel_flag.store(false, Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(1), async { + while !runtime.cancel_flag.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + release.notify_one(); + delete_task.await.unwrap().unwrap(); +} diff --git a/src-tauri/crates/agent-core/src/state/commands/session/mod.rs b/src-tauri/crates/agent-core/src/state/commands/session/mod.rs index dee3d8cf7..b52edbef9 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/mod.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/mod.rs @@ -7,6 +7,9 @@ //! Heavy logic lives in sub-modules; this file keeps the `#[tauri::command]` //! wrappers thin so Tauri's code-gen can resolve them at `commands::*`. +mod agent_org_delete; +#[cfg(test)] +mod agent_org_delete_tests; pub mod channel; mod coding; pub(crate) mod common; diff --git a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs index aa0f73733..2a66362dc 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs @@ -1,21 +1,14 @@ //! Persistence commands for session data. -use std::collections::HashSet; -use std::sync::Arc; -use std::time::Duration; - -use crate::coordination::agent_org_runs::AgentOrgRunStore; use crate::interaction::plan_approval::persistence::PlanApprovalStore; use crate::persistence::db_helpers as shared; use crate::persistence::session_snapshots; use crate::session::persistence as session_persistence; use crate::session::{SessionListFilter, SessionStatus}; use crate::state::control_flow::CancelReason; -use crate::state::{AgentAppState, AgentSession}; +use crate::state::AgentAppState; use crate::tools::file_history; use core_types::workflow::{AgentRole, LinkedSession, LinkedSessionStatus, LinkedSessionType}; -use database::db::{get_connection, with_sessions_writer}; -use rusqlite::{params, Connection, OptionalExtension}; use serde::Serialize; use super::common::review_session_ids; @@ -52,659 +45,19 @@ pub async fn agent_list_all_sessions() -> Result, String> .await } -const MAX_AGENT_ORG_DELETE_SESSIONS: usize = 1_024; -const AGENT_ORG_DELETE_STOP_TIMEOUT: Duration = Duration::from_secs(10); -const AGENT_ORG_DELETE_STOP_POLL_INTERVAL: Duration = Duration::from_millis(50); - #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct DeleteSessionReceipt { pub deleted_session_ids: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] -struct AgentOrgSessionDeleteNode { - session_id: String, - parent_session_id: Option, - status: SessionStatus, - depth: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct AgentOrgSessionDeletePlan { - run_id: String, - root_session_id: String, - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus, - sessions: Vec, -} - /// Delete a session and all related data. #[tauri::command] pub async fn agent_delete_session( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { - let planned_session_id = session_id.clone(); - let plan = tokio::task::spawn_blocking(move || { - let conn = get_connection().map_err(|err| err.to_string())?; - load_agent_org_session_delete_plan(&conn, &planned_session_id) - }) - .await - .map_err(|err| format!("session deletion planning worker failed: {err}"))??; - - let Some(plan) = plan else { - let deleted_session_id = session_id.clone(); - tokio::task::spawn_blocking(move || { - session_persistence::delete_session(&deleted_session_id).map_err(|err| err.to_string()) - }) - .await - .map_err(|err| format!("session deletion worker failed: {err}"))??; - return Ok(DeleteSessionReceipt { - deleted_session_ids: vec![session_id], - }); - }; - - let fenced_plan = tokio::task::spawn_blocking(move || establish_agent_org_delete_fence(&plan)) - .await - .map_err(|err| format!("Agent Org deletion fence worker failed: {err}"))??; - let quiesced_runtime_session_ids = if fenced_plan.run_status - == crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - { - stop_agent_org_runtime_sessions(&state, &fenced_plan).await? - } else { - wait_for_agent_org_runtime_sessions_idle(&state, &fenced_plan).await?; - HashSet::new() - }; - let root_session_id = fenced_plan.root_session_id.clone(); - let plan = tokio::task::spawn_blocking(move || { - let conn = get_connection().map_err(|err| err.to_string())?; - load_agent_org_session_delete_plan(&conn, &root_session_id)?.ok_or_else(|| { - format!( - "Refusing to delete Agent Org root {root_session_id}: ownership disappeared while stopping" - ) - }) - }) - .await - .map_err(|err| format!("Agent Org post-stop planning worker failed: {err}"))??; - if !agent_org_delete_topology_matches(&fenced_plan, &plan) { - return Err(format!( - "Refusing to delete Agent Org run {}: session hierarchy changed while stopping", - fenced_plan.run_id - )); - } - - validate_agent_org_delete_ready(&plan, &quiesced_runtime_session_ids)?; - ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; - - let receipt = tokio::task::spawn_blocking(move || { - delete_agent_org_session_hierarchy(&plan, &quiesced_runtime_session_ids) - }) - .await - .map_err(|err| format!("Agent Org session deletion worker failed: {err}"))??; - - state.remove_sessions(&receipt.deleted_session_ids).await; - if let Some(app_handle) = state.app_handle.as_ref() { - for deleted_session_id in &receipt.deleted_session_ids { - crate::bus::event_pipeline_bridge::evict_session(app_handle, deleted_session_id); - } - } - Ok(receipt) -} - -fn load_agent_org_session_delete_plan( - conn: &Connection, - root_session_id: &str, -) -> Result, String> { - let run_rows = { - let mut stmt = conn - .prepare( - "SELECT id, status - FROM agent_org_runs - WHERE root_session_id=?1 - ORDER BY id - LIMIT 2", - ) - .map_err(|err| err.to_string())?; - let rows = stmt - .query_map([root_session_id], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) - }) - .map_err(|err| err.to_string())?; - rows.collect::, _>>() - .map_err(|err| err.to_string())? - }; - - let Some((run_id, run_status_raw)) = run_rows.first() else { - return Ok(None); - }; - if run_rows.len() != 1 { - return Err(format!( - "Refusing to delete Agent Org root {root_session_id}: at least 2 runs claim the same root" - )); - } - let run_status = crate::coordination::agent_org_runs::AgentOrgRunStatus::parse(run_status_raw) - .ok_or_else(|| { - format!( - "Refusing to delete Agent Org run {run_id}: unknown run status {run_status_raw:?}" - ) - })?; - - let mut stmt = conn - .prepare( - "WITH RECURSIVE descendants( - session_id, parent_session_id, status, depth, path, cycle - ) AS ( - SELECT session_id, - parent_session_id, - status, - 0, - '/' || hex(session_id) || '/', - 0 - FROM agent_sessions - WHERE session_id=?1 - UNION ALL - SELECT child.session_id, - child.parent_session_id, - child.status, - parent.depth + 1, - parent.path || hex(child.session_id) || '/', - instr(parent.path, '/' || hex(child.session_id) || '/') > 0 - FROM agent_sessions child - JOIN descendants parent - ON child.parent_session_id=parent.session_id - WHERE parent.cycle=0 - AND parent.depth < ?3 - ) - SELECT descendant.session_id, - descendant.parent_session_id, - descendant.status, - descendant.depth, - descendant.cycle, - ( - SELECT nested.id - FROM agent_org_runs nested - WHERE nested.id<>?2 - AND nested.root_session_id=descendant.session_id - ORDER BY nested.id - LIMIT 1 - ) AS nested_run_id, - EXISTS( - SELECT 1 - FROM agent_sessions child - WHERE child.parent_session_id=descendant.session_id - ) AS has_children - FROM descendants descendant", - ) - .map_err(|err| err.to_string())?; - let rows = stmt - .query_map( - params![ - root_session_id, - run_id, - MAX_AGENT_ORG_DELETE_SESSIONS as i64 - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, String>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, bool>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, bool>(6)?, - )) - }, - ) - .map_err(|err| err.to_string())?; - - let mut sessions = Vec::new(); - let mut visited = std::collections::HashSet::new(); - for row in rows { - let (session_id, parent_session_id, status_raw, depth, cycle, nested_run_id, has_children) = - row.map_err(|err| err.to_string())?; - if cycle { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: session ancestry contains a cycle at {session_id}" - )); - } - if !visited.insert(session_id.clone()) { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: session hierarchy visits {session_id} more than once" - )); - } - if depth < 0 { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: invalid depth for {session_id}" - )); - } - let depth = usize::try_from(depth).map_err(|err| err.to_string())?; - if depth >= MAX_AGENT_ORG_DELETE_SESSIONS && has_children { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: session hierarchy exceeds {MAX_AGENT_ORG_DELETE_SESSIONS} nodes" - )); - } - if depth > 0 { - if let Some(nested_run_id) = nested_run_id { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: descendant session {session_id} is root of unsupported nested run {nested_run_id}" - )); - } - } - let status = SessionStatus::parse(&status_raw).ok_or_else(|| { - format!( - "Refusing to delete Agent Org run {run_id}: session {session_id} has unknown status {status_raw:?}" - ) - })?; - sessions.push(AgentOrgSessionDeleteNode { - session_id, - parent_session_id, - status, - depth, - }); - if sessions.len() > MAX_AGENT_ORG_DELETE_SESSIONS { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: session hierarchy exceeds {MAX_AGENT_ORG_DELETE_SESSIONS} nodes" - )); - } - } - if sessions.is_empty() - || sessions - .iter() - .all(|node| node.session_id != root_session_id) - { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: root session {root_session_id} is missing" - )); - } - let depths = sessions - .iter() - .map(|node| (node.session_id.as_str(), node.depth)) - .collect::>(); - for node in &sessions { - if node.depth == 0 { - if node.session_id != root_session_id { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: unexpected depth-zero session {}", - node.session_id - )); - } - continue; - } - let parent_session_id = node.parent_session_id.as_deref().ok_or_else(|| { - format!( - "Refusing to delete Agent Org run {run_id}: descendant session {} has no parent", - node.session_id - ) - })?; - let parent_depth = depths.get(parent_session_id).ok_or_else(|| { - format!( - "Refusing to delete Agent Org run {run_id}: descendant session {} references missing parent {parent_session_id}", - node.session_id - ) - })?; - if parent_depth.saturating_add(1) != node.depth { - return Err(format!( - "Refusing to delete Agent Org run {run_id}: descendant session {} has inconsistent depth", - node.session_id - )); - } - } - - sessions.sort_by(|left, right| { - right - .depth - .cmp(&left.depth) - .then_with(|| left.session_id.cmp(&right.session_id)) - }); - Ok(Some(AgentOrgSessionDeletePlan { - run_id: run_id.clone(), - root_session_id: root_session_id.to_string(), - run_status, - sessions, - })) -} - -fn agent_org_delete_topology_matches( - expected: &AgentOrgSessionDeletePlan, - current: &AgentOrgSessionDeletePlan, -) -> bool { - expected.run_id == current.run_id - && expected.root_session_id == current.root_session_id - && expected.sessions.len() == current.sessions.len() - && expected - .sessions - .iter() - .zip(¤t.sessions) - .all(|(left, right)| { - left.session_id == right.session_id - && left.parent_session_id == right.parent_session_id - && left.depth == right.depth - }) -} - -fn establish_agent_org_delete_fence( - expected_plan: &AgentOrgSessionDeletePlan, -) -> Result { - let (current_plan, changed) = with_sessions_writer(|| { - let mut conn = get_connection().map_err(|err| err.to_string())?; - let tx = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(|err| err.to_string())?; - let mut current_plan = - load_agent_org_session_delete_plan(&tx, &expected_plan.root_session_id)?.ok_or_else( - || { - format!( - "Refusing to delete Agent Org run {}: root ownership changed before stopping", - expected_plan.run_id - ) - }, - )?; - if !agent_org_delete_topology_matches(expected_plan, ¤t_plan) { - return Err(format!( - "Refusing to delete Agent Org run {}: session hierarchy changed before stopping", - expected_plan.run_id - )); - } - - let cancelled_run_ids = - crate::coordination::agent_org_runs::establish_conversation_delete_fence_with_connection( - &tx, - ¤t_plan.root_session_id, - )?; - let changed = match current_plan.run_status { - crate::coordination::agent_org_runs::AgentOrgRunStatus::Starting - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Running - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Paused => { - if !cancelled_run_ids.contains(¤t_plan.run_id) { - return Err(format!( - "Refusing to delete Agent Org run {}: run status changed before cancellation", - current_plan.run_id - )); - } - current_plan.run_status = - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled; - true - } - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled => false, - status if status.is_terminal() => false, - status => { - return Err(format!( - "Refusing to delete Agent Org run {}: unsupported run status {}", - current_plan.run_id, - status.as_str() - )); - } - }; - tx.commit().map_err(|err| err.to_string())?; - Ok::<_, String>((current_plan, changed)) - })?; - if changed { - crate::coordination::agent_org_run_events::notify_agent_org_run_changed( - ¤t_plan.run_id, - ); - } - Ok(current_plan) -} - -fn validate_agent_org_delete_ready( - plan: &AgentOrgSessionDeletePlan, - quiesced_runtime_session_ids: &HashSet, -) -> Result<(), String> { - if !plan.run_status.is_terminal() { - return Err(format!( - "Refusing to delete Agent Org run {}: run status is {}", - plan.run_id, - plan.run_status.as_str() - )); - } - - for node in &plan.sessions { - let allowed = node.status == SessionStatus::Idle - || node.status.is_terminal() - || (plan.run_status - == crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - && (matches!(node.status, SessionStatus::Pending | SessionStatus::Paused) - || (node.status.is_in_flight() - && quiesced_runtime_session_ids.contains(&node.session_id)))); - if !allowed { - return Err(format!( - "Refusing to delete Agent Org run {}: session {} status is {}", - plan.run_id, - node.session_id, - node.status.as_str() - )); - } - } - Ok(()) -} - -async fn agent_org_runtime_sessions( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, -) -> Vec<(String, Arc)> { - let sessions = state.sessions.lock().await; - plan.sessions - .iter() - .filter_map(|node| { - sessions - .get(&node.session_id) - .cloned() - .map(|session| (node.session_id.clone(), session)) - }) - .collect() -} - -async fn agent_org_runtime_blockers( - runtime_sessions: &[(String, Arc)], -) -> Vec { - let mut blockers = Vec::new(); - for (session_id, session) in runtime_sessions { - let scheduler_processing = session.scheduler.is_processing(); - let pending_count = session.scheduler.pending_count(); - let active_turn = session.active_turn.lock().await.is_some(); - if active_turn || scheduler_processing || pending_count > 0 { - blockers.push(format!( - "{session_id}(active_turn={active_turn},scheduler_processing={scheduler_processing},pending={pending_count})" - )); - } - } - blockers -} - -fn agent_org_submission_blockers(plan: &AgentOrgSessionDeletePlan) -> Vec { - plan.sessions - .iter() - .filter(|node| { - crate::coordination::agent_org_runs::agent_org_submission_in_progress(&node.session_id) - }) - .map(|node| format!("{}(submission_in_progress=true)", node.session_id)) - .collect() -} - -async fn stop_agent_org_runtime_sessions( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, -) -> Result, String> { - stop_agent_org_runtime_sessions_with_timeout(state, plan, AGENT_ORG_DELETE_STOP_TIMEOUT).await -} - -async fn stop_agent_org_runtime_sessions_with_timeout( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, - timeout: Duration, -) -> Result, String> { - let mut runtime_session_ids = HashSet::new(); - let deadline = tokio::time::Instant::now() + timeout; - loop { - // Initialization that held a submission lease may install its runtime - // after fencing began. Refresh the registry each pass so that late - // runtime is cancelled before its lease can let deletion continue. - let runtime_sessions = agent_org_runtime_sessions(state, plan).await; - for (session_id, session) in &runtime_sessions { - if runtime_session_ids.insert(session_id.clone()) { - session - .cancel_active_turn(CancelReason::AgentOrgDelete) - .await; - } - } - let mut blockers = agent_org_runtime_blockers(&runtime_sessions).await; - blockers.extend(agent_org_submission_blockers(plan)); - if blockers.is_empty() { - return Ok(runtime_session_ids); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "Timed out stopping Agent Org run {} before deletion: {}", - plan.run_id, - blockers.join(", ") - )); - } - tokio::time::sleep(AGENT_ORG_DELETE_STOP_POLL_INTERVAL).await; - } -} - -async fn wait_for_agent_org_runtime_sessions_idle( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, -) -> Result<(), String> { - let deadline = tokio::time::Instant::now() + AGENT_ORG_DELETE_STOP_TIMEOUT; - loop { - let runtime_sessions = agent_org_runtime_sessions(state, plan).await; - let mut blockers = agent_org_runtime_blockers(&runtime_sessions).await; - blockers.extend(agent_org_submission_blockers(plan)); - if blockers.is_empty() { - return Ok(()); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "Timed out waiting for Agent Org run {} submissions before deletion: {}", - plan.run_id, - blockers.join(", ") - )); - } - tokio::time::sleep(AGENT_ORG_DELETE_STOP_POLL_INTERVAL).await; - } -} - -async fn ensure_agent_org_runtime_sessions_idle( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, -) -> Result<(), String> { - let runtime_sessions = agent_org_runtime_sessions(state, plan).await; - let mut blockers = agent_org_runtime_blockers(&runtime_sessions).await; - blockers.extend(agent_org_submission_blockers(plan)); - if blockers.is_empty() { - Ok(()) - } else { - Err(format!( - "Refusing to delete Agent Org run {}: active Rust runtime sessions: {}", - plan.run_id, - blockers.join(", ") - )) - } -} - -fn delete_agent_org_session_hierarchy( - expected_plan: &AgentOrgSessionDeletePlan, - quiesced_runtime_session_ids: &HashSet, -) -> Result { - for node in &expected_plan.sessions { - session_persistence::prepare_session_delete(&node.session_id) - .map_err(|err| format!("prepare session {} for deletion: {err}", node.session_id))?; - } - - let outcome = with_sessions_writer(|| { - let mut conn = get_connection().map_err(|err| err.to_string())?; - let tx = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(|err| err.to_string())?; - let current_plan = load_agent_org_session_delete_plan(&tx, &expected_plan.root_session_id)? - .ok_or_else(|| { - format!( - "Refusing to delete Agent Org run {}: root ownership changed before deletion", - expected_plan.run_id - ) - })?; - if current_plan != *expected_plan { - return Err(format!( - "Refusing to delete Agent Org run {}: session hierarchy or status changed before deletion", - expected_plan.run_id - )); - } - validate_agent_org_delete_ready(¤t_plan, quiesced_runtime_session_ids)?; - - for node in &expected_plan.sessions { - session_persistence::delete_session_with_connection(&tx, &node.session_id) - .map_err(|err| format!("delete session {}: {err}", node.session_id))?; - } - let outcome = AgentOrgRunStore::delete_by_id_with_connection(&tx, &expected_plan.run_id)?; - if !outcome.deleted() { - return Err(format!( - "Refusing to commit Agent Org run {} deletion: run row disappeared during deletion", - expected_plan.run_id - )); - } - crate::coordination::agent_org_runs::remove_conversation_delete_fence_with_connection( - &tx, - &expected_plan.root_session_id, - )?; - ensure_agent_org_hierarchy_absent(&tx, expected_plan)?; - tx.commit().map_err(|err| err.to_string())?; - Ok::<_, String>(outcome) - })?; - - for node in &expected_plan.sessions { - session_persistence::finish_session_delete(&node.session_id); - } - AgentOrgRunStore::finish_delete(&expected_plan.run_id, outcome); - - Ok(DeleteSessionReceipt { - deleted_session_ids: expected_plan - .sessions - .iter() - .map(|node| node.session_id.clone()) - .collect(), - }) -} - -fn ensure_agent_org_hierarchy_absent( - conn: &Connection, - plan: &AgentOrgSessionDeletePlan, -) -> Result<(), String> { - for node in &plan.sessions { - let residual: Option = conn - .query_row( - "SELECT session_id - FROM agent_sessions - WHERE session_id=?1 OR parent_session_id=?1 - ORDER BY session_id - LIMIT 1", - [&node.session_id], - |row| row.get(0), - ) - .optional() - .map_err(|err| err.to_string())?; - if let Some(session_id) = residual { - return Err(format!( - "Refusing to commit Agent Org run {} deletion: residual session hierarchy row {session_id} references deleted session {}", - plan.run_id, node.session_id - )); - } - } - let run_exists = conn - .query_row( - "SELECT EXISTS(SELECT 1 FROM agent_org_runs WHERE id=?1)", - [&plan.run_id], - |row| row.get::<_, bool>(0), - ) - .map_err(|err| err.to_string())?; - if run_exists { - return Err(format!( - "Refusing to commit Agent Org run {} deletion: run row still exists", - plan.run_id - )); - } - Ok(()) + super::agent_org_delete::delete_session(&state, session_id).await } /// Clear all messages for a session. @@ -1073,810 +426,3 @@ fn parse_agent_role(raw: Option<&str>) -> AgentRole { _ => AgentRole::Coding, } } - -#[cfg(test)] -mod tests { - use super::*; - - fn ensure_test_schemas() { - let conn = get_connection().expect("sandbox DB"); - crate::persistence::test_schema::ensure_agent_sessions_schema(&conn); - crate::foundation::persistence::session_snapshots::ensure_tables_with(&conn) - .expect("agent session tables"); - crate::session::persistence::init(&conn).expect("unified session schema"); - crate::interaction::plan_approval::persistence::init_schema(&conn) - .expect("plan approval schema"); - crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); - project_management::lineage::schema::init_lineage_tables(&conn).expect("lineage schema"); - crate::memory::learnings::init_learnings_table(&conn).expect("learnings schema"); - database::init_shell_replay_tables(&conn).expect("shell replay schema"); - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS events ( - id TEXT PRIMARY KEY, - session_id TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS code_sessions ( - session_id TEXT PRIMARY KEY, - cli_agent_type TEXT NOT NULL, - status TEXT NOT NULL, - parent_session_id TEXT, - org_member_id TEXT, - updated_at TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS session_turn_intents ( - session_id TEXT NOT NULL, - turn_intent_id TEXT NOT NULL, - client_message_id TEXT, - org_run_id TEXT, - source TEXT NOT NULL, - status TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - PRIMARY KEY (session_id, turn_intent_id) - );", - ) - .expect("session runtime schemas"); - } - - fn seed_session_with_status(session_id: &str, parent_session_id: Option<&str>, status: &str) { - let conn = get_connection().expect("sandbox DB"); - conn.execute( - "INSERT INTO agent_sessions ( - session_id, name, status, user_input, created_at, updated_at, - session_type, parent_session_id, workspace_additional_json, - key_source - ) VALUES (?1, ?2, ?3, NULL, ?4, ?4, 'agent', ?5, '{}', 'own_key')", - rusqlite::params![ - session_id, - format!("session-{session_id}"), - status, - "2026-07-16T00:00:00Z", - parent_session_id, - ], - ) - .expect("seed session"); - } - - fn seed_session(session_id: &str, parent_session_id: Option<&str>) { - seed_session_with_status(session_id, parent_session_id, "idle"); - } - - fn seed_run_with_status(run_id: &str, root_session_id: &str, status: &str) { - let conn = get_connection().expect("sandbox DB"); - conn.execute( - "INSERT INTO agent_org_runs ( - id, org_id, coordinator_agent_id, root_session_id, - entry_mode, status, created_at, updated_at - ) VALUES (?1, 'org-delete-test', 'coordinator-agent', ?2, - 'standalone_session', ?3, ?4, ?4)", - rusqlite::params![run_id, root_session_id, status, "2026-07-16T00:00:00Z"], - ) - .expect("seed run"); - } - - fn seed_run(run_id: &str, root_session_id: &str) { - seed_run_with_status(run_id, root_session_id, "completed"); - } - - fn seed_session_owned_rows(session_id: &str) { - let conn = get_connection().expect("sandbox DB"); - conn.execute( - "INSERT INTO agent_messages ( - id, session_id, role, content, sequence, created_at - ) VALUES (?1, ?2, 'user', 'delete me', 0, ?3)", - rusqlite::params![ - format!("message-{session_id}"), - session_id, - "2026-07-16T00:00:00Z" - ], - ) - .expect("seed message"); - conn.execute( - "INSERT INTO agent_todos (session_id, content) VALUES (?1, 'delete me')", - [session_id], - ) - .expect("seed todo"); - conn.execute( - "INSERT INTO events (id, session_id) VALUES (?1, ?2)", - rusqlite::params![format!("event-{session_id}"), session_id], - ) - .expect("seed event"); - conn.execute( - "INSERT INTO session_token_usage ( - session_id, session_type, total_tokens, created_at - ) VALUES (?1, 'agent', 1, ?2)", - rusqlite::params![session_id, "2026-07-16T00:00:00Z"], - ) - .expect("seed usage"); - } - - fn seed_run_owned_rows(run_id: &str) { - let conn = get_connection().expect("sandbox DB"); - conn.execute( - "INSERT INTO agent_inbox ( - recipient_agent_id, recipient_member_id, sender_agent_id, - org_run_id, payload_kind, payload_json, created_at - ) VALUES ('worker-agent', 'worker', 'system', ?1, - 'plain', '{\"summary\":\"run history\",\"text\":\"body\"}', ?2)", - rusqlite::params![run_id, "2026-07-16T00:00:00Z"], - ) - .expect("seed run inbox history"); - conn.execute( - "INSERT INTO agent_org_tasks ( - id, org_run_id, subject, status, created_at, updated_at - ) VALUES (?1, ?2, 'delete me', 'completed', ?3, ?3)", - rusqlite::params![format!("task-{run_id}"), run_id, "2026-07-16T00:00:00Z"], - ) - .expect("seed run task history"); - } - - fn row_exists(table: &str, column: &str, value: &str) -> bool { - get_connection() - .expect("sandbox DB") - .query_row( - &format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE {column}=?1)"), - [value], - |row| row.get(0), - ) - .expect("inspect durable row") - } - - #[test] - fn session_hierarchy_delete_removes_all_rust_descendants_and_run_history() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-delete-root"; - let worker = "hierarchy-delete-worker"; - let grandchild = "hierarchy-delete-grandchild"; - let unrelated = "hierarchy-delete-unrelated"; - let unrelated_root = "hierarchy-delete-other-root"; - seed_session(root, None); - seed_session_with_status(worker, Some(root), "completed"); - seed_session_with_status(grandchild, Some(worker), "failed"); - seed_session(unrelated, None); - seed_session(unrelated_root, None); - seed_run("hierarchy-delete-run", root); - seed_run("hierarchy-delete-other-run", unrelated_root); - for session_id in [root, worker, grandchild, unrelated] { - seed_session_owned_rows(session_id); - } - seed_run_owned_rows("hierarchy-delete-run"); - seed_run_owned_rows("hierarchy-delete-other-run"); - - let conn = get_connection().expect("sandbox DB"); - let plan = load_agent_org_session_delete_plan(&conn, root) - .expect("plan hierarchy") - .expect("root owns Agent Org run"); - drop(conn); - let receipt = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) - .expect("delete completed hierarchy"); - - assert_eq!( - receipt.deleted_session_ids, - vec![grandchild.to_string(), worker.to_string(), root.to_string()] - ); - for session_id in [root, worker, grandchild] { - for table in [ - "agent_sessions", - "agent_messages", - "agent_todos", - "events", - "session_token_usage", - ] { - assert!( - !row_exists(table, "session_id", session_id), - "{table} still contains {session_id}" - ); - } - } - assert!(!row_exists("agent_org_runs", "id", "hierarchy-delete-run")); - assert!(!row_exists( - "agent_inbox", - "org_run_id", - "hierarchy-delete-run" - )); - assert!(!row_exists( - "agent_org_tasks", - "org_run_id", - "hierarchy-delete-run" - )); - assert!(row_exists("agent_sessions", "session_id", unrelated)); - assert!(row_exists("agent_messages", "session_id", unrelated)); - assert!(row_exists( - "agent_org_runs", - "id", - "hierarchy-delete-other-run" - )); - assert!(row_exists( - "agent_inbox", - "org_run_id", - "hierarchy-delete-other-run" - )); - } - - #[test] - fn session_hierarchy_delete_worker_keeps_root_and_run() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-worker-root"; - let worker = "hierarchy-worker-direct-delete"; - seed_session(root, None); - seed_session(worker, Some(root)); - seed_run("hierarchy-worker-run", root); - seed_run_owned_rows("hierarchy-worker-run"); - - let conn = get_connection().expect("sandbox DB"); - conn.execute( - "INSERT INTO agent_org_run_sessions ( - org_run_id, member_id, session_id, role, created_at - ) VALUES ('hierarchy-worker-run', 'worker', ?1, 'worker', ?2)", - rusqlite::params![worker, "2026-07-16T00:00:00Z"], - ) - .expect("seed exact worker mapping"); - assert!( - load_agent_org_session_delete_plan(&conn, worker) - .expect("plan worker") - .is_none(), - "a worker must not be promoted to hierarchy root deletion" - ); - drop(conn); - session_persistence::delete_session(worker).expect("canonical single-session deletion"); - - assert!(!row_exists("agent_sessions", "session_id", worker)); - assert!(!row_exists("agent_org_run_sessions", "session_id", worker)); - assert!(row_exists("agent_sessions", "session_id", root)); - assert!(row_exists("agent_org_runs", "id", "hierarchy-worker-run")); - assert!(row_exists( - "agent_inbox", - "org_run_id", - "hierarchy-worker-run" - )); - } - - #[test] - fn session_hierarchy_delete_fences_active_run_and_requires_quiesced_sessions() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-active-root"; - let worker = "hierarchy-active-worker"; - seed_session(root, None); - seed_session_with_status(worker, Some(root), "running"); - seed_run_with_status("hierarchy-active-run", root, "running"); - - let conn = get_connection().expect("sandbox DB"); - let plan = load_agent_org_session_delete_plan(&conn, root) - .expect("load running hierarchy") - .expect("root owns run"); - drop(conn); - let fenced = establish_agent_org_delete_fence(&plan).expect("cancel run for deletion"); - assert_eq!( - fenced.run_status, - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - ); - assert_eq!( - get_connection() - .expect("sandbox DB") - .query_row( - "SELECT status FROM agent_org_runs WHERE id='hierarchy-active-run'", - [], - |row| row.get::<_, String>(0) - ) - .expect("load fenced status"), - "cancelled" - ); - - let error = validate_agent_org_delete_ready(&fenced, &HashSet::new()) - .expect_err("unobserved running worker must fail closed"); - assert!(error.contains(worker)); - assert!(error.contains("running")); - - let quiesced = HashSet::from([worker.to_string()]); - validate_agent_org_delete_ready(&fenced, &quiesced) - .expect("a stopped live runtime may retain a stale running row"); - assert!(row_exists("agent_sessions", "session_id", root)); - assert!(row_exists("agent_sessions", "session_id", worker)); - assert!(row_exists("agent_org_runs", "id", "hierarchy-active-run")); - } - - #[test] - fn session_hierarchy_delete_blocks_resource_preflight_failures_before_database_changes() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-replay-root"; - let worker = "hierarchy-replay-worker"; - seed_session(root, None); - seed_session(worker, Some(root)); - seed_run("hierarchy-replay-run", root); - let conn = get_connection().expect("sandbox DB"); - let plan = load_agent_org_session_delete_plan(&conn, root) - .expect("plan hierarchy") - .expect("root owns run"); - drop(conn); - - let replay_root = std::env::temp_dir().join(format!( - "orgii-hierarchy-delete-replay-{}", - uuid::Uuid::new_v4() - )); - std::fs::create_dir_all(&replay_root).expect("create replay root"); - let writer = crate::tools::impls::coding::exec::shell_replay::ShellReplayWriter::create( - &replay_root, - crate::tools::impls::coding::exec::shell_replay::ShellReplayTarget::new( - worker, - "active-call", - ), - "still running", - &replay_root, - None, - ) - .expect("create active replay"); - - let error = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) - .expect_err("active replay must block hierarchy deletion"); - assert!(error.contains(worker)); - assert!(error.contains("shell replay calls are active")); - assert!(row_exists("agent_sessions", "session_id", root)); - assert!(row_exists("agent_sessions", "session_id", worker)); - assert!(row_exists("agent_org_runs", "id", "hierarchy-replay-run")); - - writer - .finalize(core_types::session_event::ShellReplayStatus::Complete, None) - .expect("finalize replay"); - - let worktree_path = replay_root.join("owned-worktree"); - let missing_repo_path = replay_root.join("missing-repository"); - std::fs::create_dir_all(&worktree_path).expect("create worktree fixture"); - get_connection() - .expect("sandbox DB") - .execute( - "UPDATE agent_sessions - SET workspace_path=?1, worktree_path=?2, base_branch='develop' - WHERE session_id=?3", - rusqlite::params![ - missing_repo_path.to_string_lossy(), - worktree_path.to_string_lossy(), - worker, - ], - ) - .expect("seed invalid worktree metadata"); - let error = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) - .expect_err("worktree validation failure must block hierarchy deletion"); - assert!(error.contains(worker)); - assert!(error.contains("repository path no longer exists")); - assert!(row_exists("agent_sessions", "session_id", root)); - assert!(row_exists("agent_sessions", "session_id", worker)); - assert!(row_exists("agent_org_runs", "id", "hierarchy-replay-run")); - - std::fs::remove_dir_all(replay_root).expect("remove replay fixture"); - } - - #[test] - fn session_hierarchy_delete_rejects_nested_agent_org_without_mutation() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let outer_root = "hierarchy-nested-outer-root"; - let inner_root = "hierarchy-nested-inner-root"; - let inner_worker = "hierarchy-nested-inner-worker"; - seed_session(outer_root, None); - seed_session(inner_root, Some(outer_root)); - seed_session(inner_worker, Some(inner_root)); - seed_run("hierarchy-nested-outer-run", outer_root); - seed_run("hierarchy-nested-inner-run", inner_root); - - let conn = get_connection().expect("sandbox DB"); - let error = load_agent_org_session_delete_plan(&conn, outer_root) - .expect_err("nested Agent Org must fail closed"); - assert!(error.contains(inner_root)); - assert!(error.contains("hierarchy-nested-inner-run")); - for session_id in [outer_root, inner_root, inner_worker] { - assert!(row_exists("agent_sessions", "session_id", session_id)); - } - assert!(row_exists( - "agent_org_runs", - "id", - "hierarchy-nested-outer-run" - )); - assert!(row_exists( - "agent_org_runs", - "id", - "hierarchy-nested-inner-run" - )); - } - - #[test] - fn session_hierarchy_delete_rejects_cycle_and_size_limit() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let cycle_root = "hierarchy-cycle-root"; - let cycle_worker = "hierarchy-cycle-worker"; - seed_session(cycle_root, Some(cycle_worker)); - seed_session(cycle_worker, Some(cycle_root)); - seed_run("hierarchy-cycle-run", cycle_root); - - let conn = get_connection().expect("sandbox DB"); - let error = load_agent_org_session_delete_plan(&conn, cycle_root) - .expect_err("cycle must fail closed"); - assert!(error.contains("cycle")); - assert!(row_exists("agent_sessions", "session_id", cycle_root)); - assert!(row_exists("agent_sessions", "session_id", cycle_worker)); - drop(conn); - - let limit_root = "hierarchy-limit-root"; - seed_session(limit_root, None); - seed_run("hierarchy-limit-run", limit_root); - let mut conn = get_connection().expect("sandbox DB"); - let tx = conn.transaction().expect("seed oversized hierarchy"); - for index in 0..MAX_AGENT_ORG_DELETE_SESSIONS { - let session_id = format!("hierarchy-limit-worker-{index:04}"); - tx.execute( - "INSERT INTO agent_sessions ( - session_id, name, status, created_at, updated_at, - session_type, parent_session_id, workspace_additional_json, - key_source - ) VALUES (?1, ?1, 'idle', ?2, ?2, 'agent', ?3, '{}', 'own_key')", - rusqlite::params![session_id, "2026-07-16T00:00:00Z", limit_root], - ) - .expect("seed worker"); - } - tx.commit().expect("commit oversized hierarchy"); - let error = load_agent_org_session_delete_plan(&conn, limit_root) - .expect_err("oversized hierarchy must fail closed"); - assert!(error.contains("exceeds")); - assert!(row_exists("agent_sessions", "session_id", limit_root)); - assert!(row_exists("agent_org_runs", "id", "hierarchy-limit-run")); - } - - #[test] - fn session_hierarchy_delete_rechecks_concurrent_structure_changes() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-recheck-root"; - let worker = "hierarchy-recheck-worker"; - seed_session(root, None); - seed_session(worker, Some(root)); - seed_run("hierarchy-recheck-run", root); - - let conn = get_connection().expect("sandbox DB"); - let plan = load_agent_org_session_delete_plan(&conn, root) - .expect("initial plan") - .expect("root owns run"); - drop(conn); - seed_session("hierarchy-recheck-late-worker", Some(root)); - - let error = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) - .expect_err("changed hierarchy must fail closed"); - assert!(error.contains("changed before deletion")); - for session_id in [root, worker, "hierarchy-recheck-late-worker"] { - assert!(row_exists("agent_sessions", "session_id", session_id)); - } - assert!(row_exists("agent_org_runs", "id", "hierarchy-recheck-run")); - } - - #[test] - fn session_hierarchy_delete_rolls_back_on_midway_database_failure() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-rollback-root"; - let worker = "hierarchy-rollback-worker"; - seed_session(root, None); - seed_session(worker, Some(root)); - seed_session_owned_rows(root); - seed_session_owned_rows(worker); - seed_run("hierarchy-rollback-run", root); - seed_run_owned_rows("hierarchy-rollback-run"); - - let conn = get_connection().expect("sandbox DB"); - let plan = load_agent_org_session_delete_plan(&conn, root) - .expect("plan hierarchy") - .expect("root owns run"); - conn.execute_batch( - "CREATE TRIGGER hierarchy_delete_abort_root - BEFORE DELETE ON agent_sessions - WHEN OLD.session_id='hierarchy-rollback-root' - BEGIN - SELECT RAISE(ABORT, 'injected hierarchy delete failure'); - END;", - ) - .expect("install failure trigger"); - drop(conn); - - let error = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) - .expect_err("trigger must abort transaction"); - assert!(error.contains("injected hierarchy delete failure")); - for session_id in [root, worker] { - for table in [ - "agent_sessions", - "agent_messages", - "agent_todos", - "events", - "session_token_usage", - ] { - assert!( - row_exists(table, "session_id", session_id), - "{table} lost {session_id} despite rollback" - ); - } - } - assert!(row_exists("agent_org_runs", "id", "hierarchy-rollback-run")); - assert!(row_exists( - "agent_inbox", - "org_run_id", - "hierarchy-rollback-run" - )); - assert!(row_exists( - "agent_org_tasks", - "org_run_id", - "hierarchy-rollback-run" - )); - } - - #[test] - fn session_hierarchy_delete_rolls_back_transaction_time_structure_changes() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-trigger-change-root"; - let worker = "hierarchy-trigger-change-worker"; - let injected = "hierarchy-trigger-change-injected"; - seed_session(root, None); - seed_session(worker, Some(root)); - seed_run("hierarchy-trigger-change-run", root); - - let conn = get_connection().expect("sandbox DB"); - let plan = load_agent_org_session_delete_plan(&conn, root) - .expect("plan hierarchy") - .expect("root owns run"); - conn.execute_batch( - "CREATE TRIGGER hierarchy_delete_insert_child - AFTER DELETE ON agent_sessions - WHEN OLD.session_id='hierarchy-trigger-change-root' - BEGIN - INSERT INTO agent_sessions ( - session_id, name, status, created_at, updated_at, - session_type, parent_session_id, workspace_additional_json, - key_source - ) VALUES ( - 'hierarchy-trigger-change-injected', - 'injected', - 'idle', - '2026-07-16T00:00:00Z', - '2026-07-16T00:00:00Z', - 'agent', - 'hierarchy-trigger-change-root', - '{}', - 'own_key' - ); - END;", - ) - .expect("install mutation trigger"); - drop(conn); - - let error = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) - .expect_err("transaction-time hierarchy mutation must abort"); - assert!(error.contains("residual session hierarchy row")); - assert!(row_exists("agent_sessions", "session_id", root)); - assert!(row_exists("agent_sessions", "session_id", worker)); - assert!(!row_exists("agent_sessions", "session_id", injected)); - assert!(row_exists( - "agent_org_runs", - "id", - "hierarchy-trigger-change-run" - )); - } - - #[tokio::test] - async fn session_hierarchy_delete_stops_active_runtime_and_discards_pending_work() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-runtime-root"; - let state = AgentAppState::new(); - let root_runtime = std::sync::Arc::new(crate::state::AgentSession::new( - root.to_string(), - crate::definitions::AgentDefinition::default(), - )); - let turn_started = std::sync::Arc::new(tokio::sync::Notify::new()); - let turn_started_for_job = std::sync::Arc::clone(&turn_started); - let runtime_for_job = std::sync::Arc::clone(&root_runtime); - root_runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Turn, - message_id: "hierarchy-runtime-processing".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-processing-intent".to_string(), - org_run_id: Some("hierarchy-runtime-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let runtime = std::sync::Arc::clone(&runtime_for_job); - let started = std::sync::Arc::clone(&turn_started_for_job); - Box::pin(async move { - runtime.begin_turn("still running".to_string()).await; - started.notify_one(); - while !runtime - .cancel_flag - .load(std::sync::atomic::Ordering::SeqCst) - { - tokio::task::yield_now().await; - } - runtime - .end_turn( - crate::session::DialogTurnState::Cancelled, - crate::session::TurnStats::default(), - ) - .await; - Err("cancelled for hierarchy deletion".to_string()) - }) - }), - }) - .await - .expect("enqueue processing work"); - tokio::time::timeout(std::time::Duration::from_secs(1), turn_started.notified()) - .await - .expect("turn starts processing"); - let pending_executed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let pending_executed_for_job = std::sync::Arc::clone(&pending_executed); - root_runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Turn, - message_id: "hierarchy-runtime-pending".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-pending-intent".to_string(), - org_run_id: Some("hierarchy-runtime-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let executed = std::sync::Arc::clone(&pending_executed_for_job); - Box::pin(async move { - executed.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(String::new()) - }) - }), - }) - .await - .expect("enqueue pending work"); - state - .sessions - .lock() - .await - .insert(root.to_string(), std::sync::Arc::clone(&root_runtime)); - let plan = AgentOrgSessionDeletePlan { - run_id: "hierarchy-runtime-run".to_string(), - root_session_id: root.to_string(), - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled, - sessions: vec![AgentOrgSessionDeleteNode { - session_id: root.to_string(), - parent_session_id: None, - status: SessionStatus::Running, - depth: 0, - }], - }; - - let quiesced = stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_secs(1), - ) - .await - .expect("active Rust runtime stops"); - assert_eq!(quiesced, HashSet::from([root.to_string()])); - assert_eq!(root_runtime.scheduler.pending_count(), 0); - assert!(!root_runtime.scheduler.is_processing()); - assert!(root_runtime.active_turn.lock().await.is_none()); - assert!(!pending_executed.load(std::sync::atomic::Ordering::SeqCst)); - validate_agent_org_delete_ready(&plan, &quiesced) - .expect("quiesced active status is safe behind cancelled fence"); - } - - #[tokio::test] - async fn session_hierarchy_delete_times_out_without_removing_runtime() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-runtime-timeout-root"; - let state = AgentAppState::new(); - let runtime = std::sync::Arc::new(crate::state::AgentSession::new( - root.to_string(), - crate::definitions::AgentDefinition::default(), - )); - let release = std::sync::Arc::new(tokio::sync::Notify::new()); - let release_for_job = std::sync::Arc::clone(&release); - runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Maintenance, - message_id: "hierarchy-runtime-timeout".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-timeout-intent".to_string(), - org_run_id: Some("hierarchy-runtime-timeout-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let release = std::sync::Arc::clone(&release_for_job); - Box::pin(async move { - release.notified().await; - Ok(String::new()) - }) - }), - }) - .await - .expect("enqueue non-cooperative maintenance"); - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while !runtime.scheduler.is_processing() { - tokio::task::yield_now().await; - } - }) - .await - .expect("maintenance starts"); - state - .sessions - .lock() - .await - .insert(root.to_string(), std::sync::Arc::clone(&runtime)); - let plan = AgentOrgSessionDeletePlan { - run_id: "hierarchy-runtime-timeout-run".to_string(), - root_session_id: root.to_string(), - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled, - sessions: vec![AgentOrgSessionDeleteNode { - session_id: root.to_string(), - parent_session_id: None, - status: SessionStatus::Running, - depth: 0, - }], - }; - - let error = stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_millis(50), - ) - .await - .expect_err("non-cooperative work must time out"); - assert!(error.contains("Timed out stopping")); - assert!(error.contains(root)); - assert!(state.get_session(root).await.is_some()); - release.notify_one(); - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while runtime.scheduler.is_processing() { - tokio::task::yield_now().await; - } - }) - .await - .expect("maintenance finishes after the timeout assertion"); - } - - #[tokio::test] - async fn session_hierarchy_delete_waits_for_agent_org_submission_lease() { - let _sandbox = test_helpers::test_env::sandbox(); - let root = "hierarchy-submission-lease-root"; - let state = AgentAppState::new(); - let plan = AgentOrgSessionDeletePlan { - run_id: "hierarchy-submission-lease-run".to_string(), - root_session_id: root.to_string(), - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled, - sessions: vec![AgentOrgSessionDeleteNode { - session_id: root.to_string(), - parent_session_id: None, - status: SessionStatus::Pending, - depth: 0, - }], - }; - let lease = crate::coordination::agent_org_runs::AgentOrgSubmissionLease::begin(root); - - let error = stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_millis(25), - ) - .await - .expect_err("deletion must wait for an initializing submission"); - assert!(error.contains("submission_in_progress=true"), "{error}"); - - drop(lease); - stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_millis(25), - ) - .await - .expect("deletion continues after the submission exits"); - } -} diff --git a/src-tauri/crates/e2e-test/src/agent_org.rs b/src-tauri/crates/e2e-test/src/agent_org.rs index a5d98a62f..232abb969 100644 --- a/src-tauri/crates/e2e-test/src/agent_org.rs +++ b/src-tauri/crates/e2e-test/src/agent_org.rs @@ -35,11 +35,9 @@ const DURABLE_INVARIANTS_PATH: &str = "/agent/test/agent-org/durable-invariants" const FIND_WORKER_SESSION_PATH: &str = "/agent/test/agent-org/find-worker-session"; const SEED_CLI_MEMBER_RUN_PATH: &str = "/agent/test/agent-org/stale-workers/seed-cli-member"; const SEED_RUST_MEMBER_RUN_PATH: &str = "/agent/test/agent-org/stale-workers/seed-run"; -const SESSION_DELETE_SNAPSHOT_PATH: &str = "/agent/test/agent-org/session-delete/snapshot"; const SESSION_DELETE_ATTEMPT_PATH: &str = "/agent/test/agent-org/session-delete/attempt"; const SUBMISSION_CONTROL_PATH: &str = "/agent/test/agent-org/submission-control"; const TASKS_SEED_PATH: &str = "/agent/test/agent-org/tasks/seed"; -const TASKS_LIST_PATH: &str = "/agent/test/agent-org/tasks/list"; const PAUSE_RUN_PATH: &str = "/agent/test/agent-org/run/pause"; const RESUME_RUN_PATH: &str = "/agent/test/agent-org/run/resume"; const SIMULATE_APP_RESTART_PATH: &str = "/agent/test/agent-org/simulate-app-restart"; @@ -255,7 +253,7 @@ pub(super) fn messages_array(resp: &serde_json::Value) -> Result<&Vec boo .and_then(serde_json::Value::as_str) == Some("completed"); - let snapshot_body = serde_json::json!({ - "session_ids": [ - root_session_id, - historical_worker_session_id, - current_worker_session_id - ], - "run_ids": [historical_run_id, org_run_id] - }); - let snapshot_before = - match post_agent_org_json(cfg, SESSION_DELETE_SNAPSHOT_PATH, snapshot_body.clone()).await { - Err(err) => return harness::print_error(label, &err), - Ok(json) => json, - }; - let tasks_before = match post_agent_org_json( - cfg, - TASKS_LIST_PATH, - serde_json::json!({ "org_run_id": org_run_id }), - ) - .await - { - Err(err) => return harness::print_error(label, &err), - Ok(json) => json, - }; - let delete_attempt = match post_agent_org_json( - cfg, - SESSION_DELETE_ATTEMPT_PATH, - serde_json::json!({ "session_id": root_session_id }), - ) - .await - { - Err(err) => return harness::print_error(label, &err), - Ok(json) => json, - }; - let snapshot_after = - match post_agent_org_json(cfg, SESSION_DELETE_SNAPSHOT_PATH, snapshot_body).await { - Err(err) => return harness::print_error(label, &err), - Ok(json) => json, - }; - let tasks_after = match post_agent_org_json( - cfg, - TASKS_LIST_PATH, - serde_json::json!({ "org_run_id": org_run_id }), - ) - .await - { - Err(err) => return harness::print_error(label, &err), - Ok(json) => json, - }; - let historical_view_after_delete = match post_agent_org_json( - cfg, - RUN_VIEW_PATH, - serde_json::json!({ "session_id": historical_worker_session_id }), - ) - .await - { - Err(err) => return harness::print_error(label, &err), - Ok(json) => json, - }; - let current_view_after_delete = match post_agent_org_json( - cfg, - RUN_VIEW_PATH, - serde_json::json!({ "session_id": current_worker_session_id }), - ) - .await - { - Err(err) => return harness::print_error(label, &err), - Ok(json) => json, - }; - let deletion_rejected = delete_attempt.get("ok").and_then(|value| value.as_bool()) - == Some(false) - && delete_attempt - .get("error") - .and_then(serde_json::Value::as_str) - .is_some_and(|error| error.contains("at least 2 runs claim the same root")); - let snapshot_has = |group: &str, id: &str| { - snapshot_before - .get(group) - .and_then(|value| value.get(id)) - .is_some_and(serde_json::Value::is_object) - }; - let snapshot_complete = [ - &root_session_id, - &historical_worker_session_id, - ¤t_worker_session_id, - ] - .iter() - .all(|id| snapshot_has("sessions", id)) - && [&historical_run_id, &org_run_id] - .iter() - .all(|id| snapshot_has("runs", id)) - && snapshot_before - .get("mappings") - .and_then(serde_json::Value::as_array) - .is_some_and(|mappings| mappings.len() == 4); - let durable_state_unchanged = snapshot_complete - && snapshot_before == snapshot_after - && tasks_before == tasks_after - && tasks_before - .get("tasks") - .and_then(serde_json::Value::as_array) - .is_some_and(|tasks| tasks.len() == 1); - let exact_ownership_after_delete = run_view_has_exact_worker( - &historical_view_after_delete, - &historical_run_id, - member_id, - &historical_worker_session_id, - ) && run_view_has_exact_worker( - ¤t_view_after_delete, - &org_run_id, - member_id, - ¤t_worker_session_id, - ); - let resume_resp = match post_agent_org_json( cfg, RESUME_RUN_PATH, @@ -3614,8 +3499,6 @@ pub async fn app_restart_transitions_running_runs_to_paused(cfg: &Config) -> boo "inv_before_restart": inv_before_restart, "inv_after_restart": inv_after_restart, "run_view_after_restart": run_view_resp, - "delete_attempt": delete_attempt, - "snapshot_before": snapshot_before, "resume": resume_resp, "inv_after_resume": inv_after_resume, }) @@ -3648,18 +3531,6 @@ pub async fn app_restart_transitions_running_runs_to_paused(cfg: &Config) -> boo "historical run stays completed", historical_run_stayed_completed, ), - ( - "production multi-run deletion is rejected", - deletion_rejected, - ), - ( - "delete rejection changes no durable rows", - durable_state_unchanged, - ), - ( - "exact mappings survive delete rejection", - exact_ownership_after_delete, - ), ("resume endpoint ok", resume_ok), ("resume transitioned=true", resume_transitioned), ( diff --git a/src-tauri/crates/e2e-test/src/agent_org_session_delete.rs b/src-tauri/crates/e2e-test/src/agent_org_session_delete.rs new file mode 100644 index 000000000..2a5258761 --- /dev/null +++ b/src-tauri/crates/e2e-test/src/agent_org_session_delete.rs @@ -0,0 +1,597 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use super::agent_org::{post_agent_org_json as post_json, unique_run_id}; +use super::config::Config; +use super::harness; + +const SEED_RUN_PATH: &str = "/agent/test/agent-org/stale-workers/seed-run"; +const SEED_RELATED_PATH: &str = "/agent/test/agent-org/session-delete/fixture/seed-related"; +const SNAPSHOT_PATH: &str = "/agent/test/agent-org/session-delete/snapshot"; +const SUPPORT_SNAPSHOT_PATH: &str = "/agent/test/agent-org/session-delete/fixture/support-snapshot"; +const DELETE_ATTEMPT_PATH: &str = "/agent/test/agent-org/session-delete/attempt"; +const FAULT_ARM_PATH: &str = "/agent/test/agent-org/session-delete/fault/arm"; +const FAULT_DISARM_PATH: &str = "/agent/test/agent-org/session-delete/fault/disarm"; +const RESTART_PATH: &str = "/agent/test/agent-org/simulate-app-restart"; +const RESUME_PATH: &str = "/agent/test/agent-org/run/resume"; +const DELETE_FAULT_ERROR: &str = "e2e_agent_org_session_delete_fault"; + +struct FixtureIds { + root: String, + history_worker: String, + live_worker: String, + unrelated_root: String, + unrelated_worker: String, +} + +impl FixtureIds { + fn new(label: &str) -> Self { + let fixture_id = unique_run_id(label); + Self { + root: format!("{fixture_id}-root"), + history_worker: format!("{fixture_id}-worker-history"), + live_worker: format!("{fixture_id}-worker-live"), + unrelated_root: format!("{fixture_id}-unrelated-root"), + unrelated_worker: format!("{fixture_id}-unrelated-worker"), + } + } + + fn target_session_ids(&self) -> Vec { + vec![ + self.root.clone(), + self.history_worker.clone(), + self.live_worker.clone(), + ] + } +} + +struct SeededFixture { + history_run: String, + live_run: String, + unrelated_run: String, +} + +impl SeededFixture { + fn target_run_ids(&self) -> Vec { + vec![self.history_run.clone(), self.live_run.clone()] + } +} + +struct ScenarioReport { + details: serde_json::Value, + checks: Vec<(&'static str, bool)>, +} + +#[derive(serde::Deserialize, serde::Serialize, PartialEq)] +struct StatusRow { + status: String, +} + +#[derive(serde::Deserialize, serde::Serialize, PartialEq)] +struct MappingRow { + org_run_id: String, + member_id: String, + session_id: String, + role: String, +} + +#[derive(Debug, Default, serde::Deserialize, serde::Serialize, PartialEq)] +struct SupportCounts { + tasks: u64, + task_events: u64, + inbox: u64, + approvals: u64, + run_progress: u64, + finality_requests: u64, + run_turn_intents: u64, + session_turn_intents: u64, +} + +impl SupportCounts { + fn expected(run_count: u64, completed_count: u64) -> Self { + Self { + tasks: run_count, + task_events: run_count, + inbox: run_count, + approvals: run_count, + run_progress: run_count, + finality_requests: completed_count, + run_turn_intents: run_count, + session_turn_intents: run_count, + } + } + + fn is_empty(&self) -> bool { + self == &Self::default() + } +} + +#[derive(serde::Deserialize, serde::Serialize, PartialEq)] +struct Snapshot { + sessions: BTreeMap>, + runs: BTreeMap>, + mappings: Vec, + #[serde(default)] + fence_count: u64, + #[serde(default)] + support_counts: SupportCounts, +} + +#[derive(serde::Deserialize)] +struct SupportSnapshot { + fence_count: u64, + support_counts: SupportCounts, +} + +impl Snapshot { + fn rows_match( + rows: &BTreeMap>, + ids: &[String], + present: bool, + ) -> bool { + ids.iter() + .all(|id| rows.get(id).is_some_and(|row| row.is_some() == present)) + } + + fn has_expected_topology(&self, ids: &FixtureIds, seeded: &SeededFixture) -> bool { + let actual = self + .mappings + .iter() + .map(|mapping| { + ( + mapping.org_run_id.as_str(), + mapping.member_id.as_str(), + mapping.session_id.as_str(), + mapping.role.as_str(), + ) + }) + .collect::>(); + let expected = BTreeSet::from([ + ( + seeded.history_run.as_str(), + "coordinator", + ids.root.as_str(), + "coordinator", + ), + ( + seeded.history_run.as_str(), + "history-member", + ids.history_worker.as_str(), + "worker", + ), + ( + seeded.live_run.as_str(), + "coordinator", + ids.root.as_str(), + "coordinator", + ), + ( + seeded.live_run.as_str(), + "live-member", + ids.live_worker.as_str(), + "worker", + ), + ]); + self.mappings.len() == actual.len() && actual == expected + } + + fn run_status(&self, run_id: &str) -> Option<&str> { + self.runs + .get(run_id) + .and_then(Option::as_ref) + .map(|row| row.status.as_str()) + } + + fn target_is_absent(&self, ids: &FixtureIds, seeded: &SeededFixture) -> bool { + Self::rows_match(&self.sessions, &ids.target_session_ids(), false) + && Self::rows_match(&self.runs, &seeded.target_run_ids(), false) + && self.mappings.is_empty() + && self.fence_count == 0 + && self.support_counts.is_empty() + } +} + +fn require_ok(response: serde_json::Value, action: &str) -> Result { + if response.get("ok").and_then(serde_json::Value::as_bool) == Some(true) { + Ok(response) + } else { + Err(format!("{action} failed: {response}")) + } +} + +async fn seed_run( + cfg: &Config, + root_session_id: &str, + run_label: &str, + run_status: &str, + worker_session_id: &str, +) -> Result { + let member_id = format!("{run_label}-member"); + let root_status = match run_status { + "paused" => "paused", + _ => "idle", + }; + let body = serde_json::json!({ + "org_id": format!("{root_session_id}:{run_label}"), + "coordinator_agent_id": "builtin:sde", + "root_session_id": root_session_id, + "root_status": root_status, + "run_status": run_status, + "workers": [{ + "member_id": member_id, + "agent_definition_id": "builtin:explore", + "session_id": worker_session_id, + "status": run_status, + }], + }); + let response = require_ok(post_json(cfg, SEED_RUN_PATH, body).await?, "seed run")?; + response + .get("org_run_id") + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("seed response omitted org_run_id: {response}")) +} + +async fn seed_related( + cfg: &Config, + root_session_id: &str, + runs: &[(&str, &str, &str, Option<&str>)], +) -> Result<(), String> { + let runs = runs + .iter() + .map(|(org_run_id, worker_session_id, member_id, final_status)| { + serde_json::json!({ + "org_run_id": org_run_id, + "worker_session_id": worker_session_id, + "member_id": member_id, + "final_status": final_status, + }) + }) + .collect::>(); + require_ok( + post_json( + cfg, + SEED_RELATED_PATH, + serde_json::json!({ "root_session_id": root_session_id, "runs": runs }), + ) + .await?, + "seed related rows", + )?; + Ok(()) +} + +async fn seed_fixture(cfg: &Config, ids: &FixtureIds) -> Result { + let history_run = seed_run(cfg, &ids.root, "history", "paused", &ids.history_worker).await?; + seed_related( + cfg, + &ids.root, + &[( + &history_run, + &ids.history_worker, + "history-member", + Some("completed"), + )], + ) + .await?; + + let live_run = seed_run(cfg, &ids.root, "live", "paused", &ids.live_worker).await?; + seed_related( + cfg, + &ids.root, + &[(&live_run, &ids.live_worker, "live-member", None)], + ) + .await?; + + let unrelated_run = seed_run( + cfg, + &ids.unrelated_root, + "unrelated", + "paused", + &ids.unrelated_worker, + ) + .await?; + + seed_related( + cfg, + &ids.unrelated_root, + &[( + &unrelated_run, + &ids.unrelated_worker, + "unrelated-member", + Some("completed"), + )], + ) + .await?; + + Ok(SeededFixture { + history_run, + live_run, + unrelated_run, + }) +} + +async fn snapshot( + cfg: &Config, + root_session_id: &str, + session_ids: &[String], + run_ids: &[String], +) -> Result { + let body = serde_json::json!({ "session_ids": session_ids, "run_ids": run_ids }); + let response = require_ok( + post_json(cfg, SNAPSHOT_PATH, body).await?, + "inspect fixture", + )?; + let mut snapshot: Snapshot = serde_json::from_value(response) + .map_err(|error| format!("invalid topology snapshot: {error}"))?; + let support = require_ok( + post_json( + cfg, + SUPPORT_SNAPSHOT_PATH, + serde_json::json!({ + "root_session_id": root_session_id, + "session_ids": session_ids, + "run_ids": run_ids, + }), + ) + .await?, + "inspect support rows", + )?; + let support: SupportSnapshot = serde_json::from_value(support) + .map_err(|error| format!("invalid support snapshot: {error}"))?; + snapshot.fence_count = support.fence_count; + snapshot.support_counts = support.support_counts; + Ok(snapshot) +} + +async fn delete_attempt(cfg: &Config, session_id: &str) -> Result { + let body = serde_json::json!({ "session_id": session_id }); + post_json(cfg, DELETE_ATTEMPT_PATH, body).await +} + +fn response_ok(response: &serde_json::Value) -> bool { + response.get("ok").and_then(serde_json::Value::as_bool) == Some(true) +} + +fn receipt_has_exact_ids(response: &serde_json::Value, expected: &[String]) -> bool { + let Some(ids) = response + .pointer("/receipt/deletedSessionIds") + .and_then(serde_json::Value::as_array) + else { + return false; + }; + let actual = ids + .iter() + .filter_map(serde_json::Value::as_str) + .collect::>(); + let expected = expected.iter().map(String::as_str).collect::>(); + ids.len() == actual.len() && actual == expected +} + +async fn cleanup_fixture(cfg: &Config, ids: &FixtureIds) -> bool { + let disarm_ok = post_json(cfg, FAULT_DISARM_PATH, serde_json::json!({})) + .await + .is_ok_and(|response| response_ok(&response)); + let mut cleanup_ok = disarm_ok; + for root_session_id in [&ids.root, &ids.unrelated_root] { + let result = delete_attempt(cfg, root_session_id).await; + if !result.as_ref().is_ok_and(response_ok) { + cleanup_ok = false; + eprintln!("[agent-org-delete-cleanup] {root_session_id}: {result:?}"); + } + } + cleanup_ok +} + +async fn run_success_scenario(cfg: &Config, ids: &FixtureIds) -> Result { + let seeded = seed_fixture(cfg, ids).await?; + let target_session_ids = ids.target_session_ids(); + let target_run_ids = seeded.target_run_ids(); + let before = snapshot(cfg, &ids.root, &target_session_ids, &target_run_ids).await?; + let unrelated_session_ids = vec![ids.unrelated_root.clone(), ids.unrelated_worker.clone()]; + let unrelated_run_ids = vec![seeded.unrelated_run.clone()]; + let unrelated_before = snapshot( + cfg, + &ids.unrelated_root, + &unrelated_session_ids, + &unrelated_run_ids, + ) + .await?; + + let deletion = delete_attempt(cfg, &ids.root).await?; + let after = snapshot(cfg, &ids.root, &target_session_ids, &target_run_ids).await?; + let unrelated_after = snapshot( + cfg, + &ids.unrelated_root, + &unrelated_session_ids, + &unrelated_run_ids, + ) + .await?; + let repeated_deletion = delete_attempt(cfg, &ids.root).await?; + let restart = post_json(cfg, RESTART_PATH, serde_json::json!({})).await?; + let after_restart = snapshot(cfg, &ids.root, &target_session_ids, &target_run_ids).await?; + + let fixture_complete = Snapshot::rows_match(&before.sessions, &target_session_ids, true) + && Snapshot::rows_match(&before.runs, &target_run_ids, true) + && before.has_expected_topology(ids, &seeded) + && before.fence_count == 0 + && before.support_counts == SupportCounts::expected(2, 1) + && Snapshot::rows_match(&unrelated_before.sessions, &unrelated_session_ids, true) + && Snapshot::rows_match(&unrelated_before.runs, &unrelated_run_ids, true) + && unrelated_before.fence_count == 0 + && unrelated_before.support_counts == SupportCounts::expected(1, 1); + let receipt_exact = + response_ok(&deletion) && receipt_has_exact_ids(&deletion, &target_session_ids); + let isolation_and_idempotency = + unrelated_before == unrelated_after && response_ok(&repeated_deletion); + let restart_safe = response_ok(&restart) && after_restart.target_is_absent(ids, &seeded); + + Ok(ScenarioReport { + details: serde_json::json!({ + "deletion": deletion, + "after": after, + "unrelated_after": unrelated_after, + "repeated_deletion": repeated_deletion, + "restart": restart, + "after_restart": after_restart, + }), + checks: vec![ + ("fixture has two runs and support rows", fixture_complete), + ("production receipt has exact session IDs", receipt_exact), + ("target state is gone", after.target_is_absent(ids, &seeded)), + ("isolation and idempotency hold", isolation_and_idempotency), + ("restart does not resurrect target state", restart_safe), + ], + }) +} + +async fn run_rollback_retry_scenario( + cfg: &Config, + ids: &FixtureIds, +) -> Result { + let seeded = seed_fixture(cfg, ids).await?; + let target_session_ids = ids.target_session_ids(); + let target_run_ids = seeded.target_run_ids(); + let unrelated_session_ids = vec![ids.unrelated_root.clone(), ids.unrelated_worker.clone()]; + let unrelated_run_ids = vec![seeded.unrelated_run.clone()]; + let before = snapshot(cfg, &ids.root, &target_session_ids, &target_run_ids).await?; + let unrelated_before = snapshot( + cfg, + &ids.unrelated_root, + &unrelated_session_ids, + &unrelated_run_ids, + ) + .await?; + + let arm_body = serde_json::json!({ "root_session_id": ids.root }); + let arm = post_json(cfg, FAULT_ARM_PATH, arm_body).await?; + // The production attempt endpoint deliberately owns no fault-fixture + // cleanup. Always issue the disarm request after the first attempt, even + // when the attempt itself returns a transport error. + let failed_deletion_result = delete_attempt(cfg, &ids.root).await; + let disarm_result = post_json(cfg, FAULT_DISARM_PATH, serde_json::json!({})).await; + let failed_deletion = failed_deletion_result?; + let disarm = disarm_result?; + let after_failure = snapshot(cfg, &ids.root, &target_session_ids, &target_run_ids).await?; + let restart = post_json(cfg, RESTART_PATH, serde_json::json!({})).await?; + let resume_body = serde_json::json!({ "org_run_id": seeded.live_run }); + let resume = post_json(cfg, RESUME_PATH, resume_body).await?; + let retry = delete_attempt(cfg, &ids.root).await?; + let after_retry = snapshot(cfg, &ids.root, &target_session_ids, &target_run_ids).await?; + let restart_after_retry = post_json(cfg, RESTART_PATH, serde_json::json!({})).await?; + let after_restart = snapshot(cfg, &ids.root, &target_session_ids, &target_run_ids).await?; + let unrelated_after = snapshot( + cfg, + &ids.unrelated_root, + &unrelated_session_ids, + &unrelated_run_ids, + ) + .await?; + + let fault_error = !response_ok(&failed_deletion) + && failed_deletion + .get("error") + .and_then(serde_json::Value::as_str) + .is_some_and(|error| error.contains(DELETE_FAULT_ERROR)); + let durable_rows_rolled_back = + Snapshot::rows_match(&after_failure.sessions, &target_session_ids, true) + && Snapshot::rows_match(&after_failure.runs, &target_run_ids, true) + && after_failure.has_expected_topology(ids, &seeded) + && before.fence_count == 0 + && before.support_counts == SupportCounts::expected(2, 1) + && after_failure.fence_count == 1 + && after_failure.support_counts == before.support_counts; + let run_states_are_safe = after_failure.run_status(&seeded.history_run) == Some("completed") + && after_failure.run_status(&seeded.live_run) == Some("cancelled"); + let resume_rejected_by_fence = !response_ok(&resume) + && resume + .get("error") + .and_then(serde_json::Value::as_str) + .is_some_and(|error| error.contains("conversation_deleting")); + let retry_receipt_exact = + response_ok(&retry) && receipt_has_exact_ids(&retry, &target_session_ids); + let restart_is_fenced = response_ok(&restart) && resume_rejected_by_fence; + let retry_is_complete = retry_receipt_exact + && after_retry.target_is_absent(ids, &seeded) + && response_ok(&restart_after_retry) + && after_restart.target_is_absent(ids, &seeded); + + Ok(ScenarioReport { + details: serde_json::json!({ + "failed_deletion": failed_deletion, + "disarm": disarm, + "after_failure": after_failure, + "restart": restart, + "resume": resume, + "retry": retry, + "after_retry": after_retry, + "restart_after_retry": restart_after_retry, + "after_restart": after_restart, + "unrelated_after": unrelated_after, + }), + checks: vec![ + ( + "fault fires and explicit disarm succeeds", + response_ok(&arm) && fault_error && response_ok(&disarm), + ), + ( + "rollback preserves exact owned rows", + durable_rows_rolled_back, + ), + ("live run remains safely cancelled", run_states_are_safe), + ("restart keeps resume fenced", restart_is_fenced), + ("retry removes exact target state", retry_is_complete), + ( + "unrelated fixture is unchanged", + unrelated_before == unrelated_after, + ), + ], + }) +} + +async fn finish_scenario( + cfg: &Config, + label: &str, + ids: &FixtureIds, + result: Result, +) -> bool { + let cleanup_ok = cleanup_fixture(cfg, ids).await; + match result { + Err(error) => { + let _ = harness::print_error(label, &error); + false + } + Ok(report) => { + let passed = harness::print_result(label, &report.details.to_string(), &report.checks); + if !cleanup_ok { + eprintln!("[{label}] isolated fixture cleanup failed"); + } + passed && cleanup_ok + } + } +} + +async fn run_registered(cfg: &Config, label: &str, rollback_retry: bool) -> bool { + let ids = FixtureIds::new(label); + let result = if rollback_retry { + run_rollback_retry_scenario(cfg, &ids).await + } else { + run_success_scenario(cfg, &ids).await + }; + finish_scenario(cfg, &format!("agent-org-{label}"), &ids, result).await +} + +pub async fn multi_run_root_delete_production_command(cfg: &Config) -> bool { + run_registered(cfg, "multi-run-root-delete-production-command", false).await +} + +pub async fn multi_run_root_delete_rollback_retry(cfg: &Config) -> bool { + run_registered(cfg, "multi-run-root-delete-rollback-retry", true).await +} + +pub async fn cleanup_fault_fixture(cfg: &Config) -> Result<(), String> { + let response = post_json(cfg, FAULT_DISARM_PATH, serde_json::json!({})).await?; + if response_ok(&response) { + Ok(()) + } else { + Err(format!("delete fault cleanup failed: {response}")) + } +} diff --git a/src-tauri/crates/e2e-test/src/main.rs b/src-tauri/crates/e2e-test/src/main.rs index 98f491096..5bbd474b3 100644 --- a/src-tauri/crates/e2e-test/src/main.rs +++ b/src-tauri/crates/e2e-test/src/main.rs @@ -33,6 +33,7 @@ //! More detail: `.cursor/skills/e2e-testing/SKILL.md`. mod agent_org; +mod agent_org_session_delete; mod agent_org_tasks_and_exec_mode; mod channel; mod config; @@ -693,6 +694,16 @@ fn all_scenarios() -> Vec { "agent-org-app-restart-transitions-running-runs-to-paused", agent_org::app_restart_transitions_running_runs_to_paused ), + scenario!( + "agent-org", + "agent-org-multi-run-root-delete-production-command", + agent_org_session_delete::multi_run_root_delete_production_command + ), + scenario!( + "agent-org", + "agent-org-multi-run-root-delete-rollback-retry", + agent_org_session_delete::multi_run_root_delete_rollback_retry + ), scenario!( "agent-org", "agent-org-run-view-task-counts-split-queued-active", @@ -1146,6 +1157,9 @@ async fn main() { Ok(count) => println!("[e2e-cleanup] Scrubbed {count} residual Agent Org run(s)"), Err(err) => eprintln!("[e2e-cleanup] Agent Org startup sweep failed: {err}"), } + if let Err(err) = agent_org_session_delete::cleanup_fault_fixture(&cfg).await { + eprintln!("[e2e-cleanup] Agent Org delete-fault startup cleanup failed: {err}"); + } let selected: Vec<&ScenarioDef> = if let Some(pos) = args.iter().position(|a| a == "--scenario") { @@ -1189,6 +1203,9 @@ async fn main() { { eprintln!("[e2e-cleanup] Agent Org scenario sweep failed: {err}"); } + if let Err(err) = agent_org_session_delete::cleanup_fault_fixture(&cfg).await { + eprintln!("[e2e-cleanup] Agent Org delete-fault scenario cleanup failed: {err}"); + } let elapsed = start.elapsed(); println!(" Time: {:.1}s", elapsed.as_secs_f64()); diff --git a/src-tauri/src/api/agent/mod.rs b/src-tauri/src/api/agent/mod.rs index e651bdad8..a46a27002 100644 --- a/src-tauri/src/api/agent/mod.rs +++ b/src-tauri/src/api/agent/mod.rs @@ -853,6 +853,22 @@ pub fn create_routes() -> Router { "/test/agent-org/session-delete/attempt", post(test::agent_org::test_agent_org_session_delete_attempt), ) + .route( + "/test/agent-org/session-delete/fixture/seed-related", + post(test::agent_org_delete::seed_related_handler), + ) + .route( + "/test/agent-org/session-delete/fixture/support-snapshot", + post(test::agent_org_delete::support_snapshot_handler), + ) + .route( + "/test/agent-org/session-delete/fault/arm", + post(test::agent_org_delete::arm_fault_handler), + ) + .route( + "/test/agent-org/session-delete/fault/disarm", + post(test::agent_org_delete::disarm_fault_handler), + ) .route( "/test/agent-org/submission-control", post(test_agent_org_submission_control), diff --git a/src-tauri/src/api/agent/test/agent_org_delete.rs b/src-tauri/src/api/agent/test/agent_org_delete.rs new file mode 100644 index 000000000..c90574856 --- /dev/null +++ b/src-tauri/src/api/agent/test/agent_org_delete.rs @@ -0,0 +1,481 @@ +//! Debug-only fixtures for Agent Org conversation deletion. +//! +//! The SQL fault is deliberately narrow: one fixed trigger and one exact root +//! session carrying the reserved E2E prefix. The E2E runner explicitly +//! disarms it immediately after the faulted production deletion attempt and +//! again during fixture cleanup. + +use std::collections::HashSet; + +use agent_core::coordination::agent_org_plan_approvals::AgentOrgPlanApprovalStore; +use axum::Json; +use rusqlite::{params, Connection, OptionalExtension}; + +const E2E_FIXTURE_PREFIX: &str = "e2e-agent-org-fixture:"; +const SIDEBAR_FIXTURE_PREFIX: &str = "sdeagent-e2e-delete-"; +const MAX_FIXTURE_IDS: usize = 16; +const MAX_FIXTURE_ID_BYTES: usize = 256; +const DELETE_FAULT_TABLE: &str = "e2e_agent_org_session_delete_fault"; +const DELETE_FAULT_TRIGGER: &str = "e2e_agent_org_session_delete_abort"; +const DELETE_FAULT_ERROR: &str = "e2e_agent_org_session_delete_fault"; +type FixtureError = Box; +type FixtureResult = Result; + +#[derive(serde::Deserialize)] +pub struct RelatedFixtureRequest { + root_session_id: String, + runs: Vec, + root_final_status: Option, +} + +#[derive(serde::Deserialize)] +struct RelatedRunRequest { + org_run_id: String, + worker_session_id: String, + member_id: String, + final_status: Option, +} + +#[derive(serde::Deserialize)] +pub struct SessionRequest { + #[serde(alias = "root_session_id")] + session_id: String, +} + +#[derive(serde::Deserialize)] +pub struct SupportSnapshotRequest { + root_session_id: String, + run_ids: Vec, + session_ids: Vec, +} + +fn validate_fixture_session_id(session_id: &str) -> FixtureResult<()> { + if !session_id.starts_with(E2E_FIXTURE_PREFIX) + && !session_id.starts_with(SIDEBAR_FIXTURE_PREFIX) + { + return Err(format!( + "session_id must start with {E2E_FIXTURE_PREFIX:?} or {SIDEBAR_FIXTURE_PREFIX:?}" + ) + .into()); + } + let safe = session_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'-' | b'.')); + if session_id.len() > MAX_FIXTURE_ID_BYTES || !safe { + return Err("session_id contains unsafe characters or is too long".into()); + } + Ok(()) +} + +fn validate_fixture_id_list( + ids: &[String], + kind: &str, + require_fixture_prefix: bool, +) -> FixtureResult<()> { + if ids.is_empty() || ids.len() > MAX_FIXTURE_IDS { + return Err(format!("{kind} must contain between 1 and {MAX_FIXTURE_IDS} items").into()); + } + let mut unique = HashSet::new(); + for id in ids { + if require_fixture_prefix { + validate_fixture_session_id(id)?; + } else { + let safe = id.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'-' | b'.') + }); + if id.is_empty() || id.len() > MAX_FIXTURE_ID_BYTES || !safe { + return Err(format!("{kind} contains an unsafe ID").into()); + } + } + if !unique.insert(id) { + return Err(format!("{kind} must not contain duplicate IDs").into()); + } + } + Ok(()) +} + +async fn blocking( + work: impl FnOnce() -> FixtureResult + Send + 'static, +) -> FixtureResult { + tokio::task::spawn_blocking(work) + .await + .map_err(FixtureError::from)? +} + +async fn blocking_json( + work: impl FnOnce() -> FixtureResult + Send + 'static, +) -> Json { + match blocking(work).await { + Err(error) => Json(serde_json::json!({ "ok": false, "error": error.to_string() })), + Ok(value) => Json(value), + } +} + +struct RelatedFixtureRun { + run_id: String, + worker_session_id: String, + member_id: String, + status: String, +} + +fn load_related_fixture_runs( + root_session_id: &str, + requested: Vec, +) -> FixtureResult> { + let conn = database::db::get_connection()?; + let mut seen_runs = HashSet::new(); + let mut seen_workers = HashSet::new(); + requested + .into_iter() + .map(|request| { + let RelatedRunRequest { + org_run_id: run_id, + worker_session_id, + member_id, + final_status, + } = request; + if !seen_runs.insert(run_id.clone()) || !seen_workers.insert(worker_session_id.clone()) + { + return Err("runs must not repeat a run or worker session".into()); + } + validate_fixture_session_id(&worker_session_id)?; + let persisted_status = conn + .query_row( + "SELECT runs.status FROM agent_org_runs runs JOIN agent_org_run_sessions sessions ON sessions.org_run_id=runs.id WHERE runs.id=?1 AND runs.root_session_id=?2 AND runs.org_id LIKE ?3 AND sessions.session_id=?4 AND sessions.member_id=?5 AND sessions.role='worker'", + params![ + run_id, + root_session_id, + format!("{E2E_FIXTURE_PREFIX}%"), + worker_session_id, + member_id, + ], + |row| row.get::<_, String>(0), + ) + .optional()? + .ok_or("run and worker are not an exact E2E ownership fixture")?; + let status = match final_status.as_deref() { + None => persisted_status, + Some("completed") => "completed".to_string(), + Some(_) => return Err("final_status may only be completed".into()), + }; + Ok(RelatedFixtureRun { + run_id, + worker_session_id, + member_id, + status, + }) + }) + .collect() +} + +fn seed_related_rows( + root_session_id: &str, + fixture_runs: &[RelatedFixtureRun], + root_final_status: Option<&str>, +) -> FixtureResult<()> { + database::db::with_sessions_writer(|| -> FixtureResult<()> { + let mut conn = database::db::get_connection()?; + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let now = chrono::Utc::now().to_rfc3339(); + for (index, fixture) in fixture_runs.iter().enumerate() { + let suffix = uuid::Uuid::new_v4(); + let task_id = format!("e2e-delete-task-{suffix}"); + let plan_path = AgentOrgPlanApprovalStore::managed_plan_path_for_session( + &fixture.worker_session_id, + &format!("e2e-delete-{index}.plan.md"), + )? + .to_string_lossy() + .into_owned(); + let completed = fixture.status == "completed"; + let task_status = if completed { "completed" } else { "pending" }; + tx.execute( + "INSERT INTO agent_org_tasks (id,org_run_id,subject,owner,status,created_at,updated_at) VALUES (?1,?2,?3,?4,?5,?6,?6)", + params![ + task_id, + fixture.run_id, + format!("Multi-run delete fixture {index}"), + fixture.member_id, + task_status, + now, + ], + )?; + tx.execute( + "INSERT INTO agent_org_task_events (id,org_run_id,task_id,event_type,next_owner,next_status,actor_member_id,created_at) VALUES (?1,?2,?3,'created',?4,?5,'coordinator',?6)", + params![ + format!("e2e-delete-task-event-{suffix}"), + fixture.run_id, + task_id, + fixture.member_id, + task_status, + now, + ], + )?; + tx.execute( + "INSERT INTO agent_inbox (recipient_agent_id,recipient_member_id,sender_agent_id,org_run_id,payload_kind,payload_json,created_at,display_text) VALUES ('builtin:explore',?1,'system',?2,'plain',?3,?4,'E2E deletion fixture')", + params![ + fixture.member_id, + fixture.run_id, + r#"{"kind":"plain","summary":"E2E deletion fixture","text":"Delete atomically."}"#, + now, + ], + )?; + let approval_status = if completed { "approved" } else { "pending" }; + let resolved_at = completed.then_some(now.as_str()); + tx.execute( + "INSERT INTO agent_org_plan_approvals (approval_id,plan_revision_id,request_id,org_run_id,source_task_id,source_member_id,source_session_id,root_session_id,policy,status,plan_title,plan_path,plan_content,decision_by,feedback,created_at,resolved_at) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,'coordinator',?9,'E2E deletion fixture',?10,'# E2E deletion fixture',?11,NULL,?12,?13)", + params![ + format!("e2e-delete-approval-{suffix}"), + format!("e2e-delete-revision-{suffix}"), + format!("e2e-delete-request-{suffix}"), + fixture.run_id, task_id, fixture.member_id, fixture.worker_session_id, + root_session_id, approval_status, plan_path, + completed.then_some("system"), + now, resolved_at, + ], + )?; + tx.execute( + "UPDATE agent_org_run_progress SET work_revision=?2,coordinator_presented_work_revision=?2,coordinator_observed_work_revision=?2,completion_requested=?3,completion_requested_at=?4,completion_requested_work_revision=?5,completion_summary=?6,updated_at=?7 WHERE org_run_id=?1", + params![ + fixture.run_id, + (index + 1) as i64, + i64::from(completed), + resolved_at, + resolved_at.map(|_| (index + 1) as i64), + resolved_at.map(|_| "E2E historical completion"), + now, + ], + )?; + tx.execute( + "UPDATE agent_org_runs SET status=?2,updated_at=?3 WHERE id=?1", + params![fixture.run_id, fixture.status, now], + )?; + if completed { + tx.execute( + "UPDATE agent_sessions + SET status='completed',updated_at=?2 + WHERE session_id IN ( + SELECT session_id FROM agent_org_run_sessions + WHERE org_run_id=?1 AND role='worker' + )", + params![fixture.run_id, now], + )?; + } + for (session_id, kind, run_id) in [ + (root_session_id, "run", Some(fixture.run_id.as_str())), + (fixture.worker_session_id.as_str(), "session", None), + ] { + tx.execute( + "INSERT INTO session_turn_intents (session_id,turn_intent_id,client_message_id,org_run_id,source,status,created_at,updated_at) VALUES (?1,?2,NULL,?3,'agent_org','completed',?4,?4)", + params![ + session_id, + format!("e2e-delete-{kind}-intent-{suffix}"), + run_id, + now + ], + )?; + } + } + match root_final_status { + None => {} + Some("completed") => { + tx.execute( + "UPDATE agent_sessions SET status='completed',updated_at=?2 WHERE session_id=?1", + params![root_session_id, now], + )?; + } + Some(_) => return Err("root_final_status may only be completed".into()), + } + Ok(tx.commit()?) + }) +} + +pub async fn seed_related_handler( + Json(body): Json, +) -> Json { + blocking_json(move || { + validate_fixture_session_id(&body.root_session_id)?; + if body.runs.is_empty() || body.runs.len() > MAX_FIXTURE_IDS { + return Err(format!("runs must contain between 1 and {MAX_FIXTURE_IDS} items").into()); + } + let runs = load_related_fixture_runs(&body.root_session_id, body.runs)?; + seed_related_rows( + &body.root_session_id, + &runs, + body.root_final_status.as_deref(), + )?; + Ok(serde_json::json!({ + "ok": true, + "seeded_run_count": runs.len(), + })) + }) + .await +} + +fn validate_snapshot_run_ids( + conn: &Connection, + root_session_id: &str, + run_ids: &[String], +) -> FixtureResult<()> { + for run_id in run_ids { + let owner = conn + .query_row( + "SELECT root_session_id,org_id FROM agent_org_runs WHERE id=?1", + [run_id], + |row| Ok((row.get::<_, Option>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + if owner.is_some_and(|(root, org_id)| { + root.as_deref() != Some(root_session_id) || !org_id.starts_with(E2E_FIXTURE_PREFIX) + }) { + return Err("run_ids contains a run outside the exact E2E fixture root".into()); + } + } + Ok(()) +} + +fn count_run_rows( + conn: &Connection, + table: &str, + predicate: &str, + run_ids: &[String], +) -> FixtureResult { + // Count by the exact reserved fixture IDs after ownership validation. A + // join through agent_org_runs would hide orphan support rows after delete. + let sql = format!( + "SELECT COUNT(*) FROM {table} row + WHERE row.org_run_id=?1 {predicate}" + ); + let mut stmt = conn.prepare(&sql)?; + run_ids.iter().try_fold(0_i64, |total, run_id| { + let count = stmt.query_row([run_id], |row| row.get::<_, i64>(0))?; + Ok(total + count) + }) +} + +fn support_snapshot(body: SupportSnapshotRequest) -> FixtureResult { + validate_fixture_session_id(&body.root_session_id)?; + validate_fixture_id_list(&body.run_ids, "run_ids", false)?; + validate_fixture_id_list(&body.session_ids, "session_ids", true)?; + if !body + .session_ids + .iter() + .any(|id| id == &body.root_session_id) + { + return Err("session_ids must include root_session_id".into()); + } + + let conn = database::db::get_connection()?; + validate_snapshot_run_ids(&conn, &body.root_session_id, &body.run_ids)?; + let count = + |table: &str, predicate: &str| count_run_rows(&conn, table, predicate, &body.run_ids); + let session_turn_intents = + body.session_ids + .iter() + .try_fold(0_i64, |total, session_id| -> FixtureResult { + let count = conn.query_row( + "SELECT COUNT(*) FROM session_turn_intents + WHERE session_id=?1 AND org_run_id IS NULL", + [session_id], + |row| row.get::<_, i64>(0), + )?; + Ok(total + count) + })?; + let fence_count = conn.query_row( + "SELECT COUNT(*) FROM agent_org_conversation_delete_fences + WHERE root_session_id=?1", + [&body.root_session_id], + |row| row.get::<_, i64>(0), + )?; + + Ok(serde_json::json!({ + "ok": true, + "fence_count": fence_count, + "support_counts": { + "tasks": count("agent_org_tasks", "")?, + "task_events": count("agent_org_task_events", "")?, + "inbox": count("agent_inbox", "")?, + "approvals": count("agent_org_plan_approvals", "")?, + "run_progress": count("agent_org_run_progress", "")?, + "finality_requests": count( + "agent_org_run_progress", + "AND row.completion_requested=1", + )?, + "run_turn_intents": count("session_turn_intents", "")?, + "session_turn_intents": session_turn_intents, + }, + })) +} + +/// Inspect only rows reachable from reserved E2E fixture IDs. The production +/// deletion command remains the sole mutation path used by the runner. +pub async fn support_snapshot_handler( + Json(body): Json, +) -> Json { + blocking_json(move || support_snapshot(body)).await +} + +fn arm_delete_fault(root_session_id: &str) -> FixtureResult<()> { + validate_fixture_session_id(root_session_id)?; + database::db::with_sessions_writer(|| -> FixtureResult<()> { + let mut conn = database::db::get_connection()?; + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let eligible: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM agent_org_runs WHERE root_session_id=?1 AND org_id LIKE ?2)", + params![root_session_id, format!("{E2E_FIXTURE_PREFIX}%")], + |row| row.get(0), + )?; + if !eligible { + return Err("fault target is not an exact persisted E2E Agent Org root".into()); + } + tx.execute_batch(&format!( + "CREATE TABLE IF NOT EXISTS {DELETE_FAULT_TABLE} ( + root_session_id TEXT PRIMARY KEY + ); + DROP TRIGGER IF EXISTS {DELETE_FAULT_TRIGGER}; + DELETE FROM {DELETE_FAULT_TABLE};" + ))?; + tx.execute( + &format!("INSERT INTO {DELETE_FAULT_TABLE} (root_session_id) VALUES (?1)"), + [root_session_id], + )?; + tx.execute_batch(&format!( + "CREATE TRIGGER {DELETE_FAULT_TRIGGER} + BEFORE DELETE ON agent_sessions + WHEN EXISTS ( + SELECT 1 FROM {DELETE_FAULT_TABLE} + WHERE root_session_id=OLD.session_id + ) + BEGIN + SELECT RAISE(ABORT, '{DELETE_FAULT_ERROR}'); + END;" + ))?; + Ok(tx.commit()?) + }) +} + +fn disarm_delete_fault() -> FixtureResult<()> { + database::db::with_sessions_writer(|| -> FixtureResult<()> { + let conn = database::db::get_connection()?; + Ok(conn.execute_batch(&format!( + "DROP TRIGGER IF EXISTS {DELETE_FAULT_TRIGGER}; + DROP TABLE IF EXISTS {DELETE_FAULT_TABLE};" + ))?) + }) +} + +pub async fn arm_fault_handler(Json(body): Json) -> Json { + blocking_json(move || { + arm_delete_fault(&body.session_id)?; + Ok(serde_json::json!({ "ok": true })) + }) + .await +} + +/// Idempotent runner cleanup for an interrupted fault scenario. +pub async fn disarm_fault_handler() -> Json { + blocking_json(|| { + disarm_delete_fault()?; + Ok(serde_json::json!({ "ok": true })) + }) + .await +} diff --git a/src-tauri/src/api/agent/test/mod.rs b/src-tauri/src/api/agent/test/mod.rs index d5670531a..dd38d7e05 100644 --- a/src-tauri/src/api/agent/test/mod.rs +++ b/src-tauri/src/api/agent/test/mod.rs @@ -7,6 +7,7 @@ //! `#[cfg(debug_assertions)]`, so no inner gating is needed here. pub mod agent_org; +pub mod agent_org_delete; pub mod cli; pub mod core; pub mod desktop; diff --git a/src/app/root/e2e/helpers/sessionHelpers/seeders.ts b/src/app/root/e2e/helpers/sessionHelpers/seeders.ts index 76485dc91..43a46e16d 100644 --- a/src/app/root/e2e/helpers/sessionHelpers/seeders.ts +++ b/src/app/root/e2e/helpers/sessionHelpers/seeders.ts @@ -22,7 +22,9 @@ import { } from "@src/store/session/planApprovalAtom"; import { type Session, + sessionPaginationAtom, sessionsAtom, + syncSessionWithNativeRosters, upsertSession, } from "@src/store/session/sessionAtom"; import { updateShellProcessAtom } from "@src/store/session/shellProcessAtom"; @@ -158,6 +160,7 @@ export function createSessionSeederHelpers(store: E2EStore) { repoPath?: string; status?: string; orgId?: string; + agentOrgId?: string; touchedFiles?: string[]; }): Promise> => { try { @@ -181,6 +184,7 @@ export function createSessionSeederHelpers(store: E2EStore) { user_input: input.name ?? existing?.user_input ?? input.sessionId, repoPath: input.repoPath ?? existing?.repoPath, orgId: input.orgId ?? existing?.orgId, + agentOrgId: input.agentOrgId ?? existing?.agentOrgId, touchedFiles: input.touchedFiles ?? existing?.touchedFiles, filesChanged: input.touchedFiles?.length ?? existing?.filesChanged ?? undefined, @@ -188,6 +192,9 @@ export function createSessionSeederHelpers(store: E2EStore) { is_active: true, }; upsertSession(session); + store.set(sessionPaginationAtom, (pagination) => + syncSessionWithNativeRosters(pagination, session) + ); return { ok: true, sessionId: input.sessionId }; } catch (err) { return asError(err); diff --git a/src/app/root/e2e/types.ts b/src/app/root/e2e/types.ts index f16f6b095..76a3faf58 100644 --- a/src/app/root/e2e/types.ts +++ b/src/app/root/e2e/types.ts @@ -579,6 +579,7 @@ export interface E2EHelpers { repoPath?: string; status?: string; orgId?: string; + agentOrgId?: string; touchedFiles?: string[]; }) => Promise>; openWorkManagementTab: () => Promise>; diff --git a/tests/e2e/specs/core/agent-org-session-delete-ui.spec.mjs b/tests/e2e/specs/core/agent-org-session-delete-ui.spec.mjs index 56901a495..c5411adf8 100644 --- a/tests/e2e/specs/core/agent-org-session-delete-ui.spec.mjs +++ b/tests/e2e/specs/core/agent-org-session-delete-ui.spec.mjs @@ -43,47 +43,68 @@ async function seedHierarchy({ const rootSessionId = `sdeagent-e2e-delete-${label}-root-${RUN_ID}`; const firstWorkerId = `sdeagent-e2e-delete-${label}-worker-a-${RUN_ID}`; const secondWorkerId = `sdeagent-e2e-delete-${label}-worker-b-${RUN_ID}`; + const orgId = `e2e-agent-org-fixture:ui-delete-${label}-${RUN_ID}`; + const materializedWorkerStatus = + workerStatus === "completed" || workerStatus === "paused" + ? "pending" + : workerStatus; const workers = [ { session_id: firstWorkerId, member_id: `${label}-worker-a`, agent_definition_id: "builtin:sde", - status: workerStatus, + status: materializedWorkerStatus, }, ]; if (nested) { workers.push({ session_id: secondWorkerId, - parent_session_id: firstWorkerId, member_id: `${label}-worker-b`, agent_definition_id: "builtin:sde", - status: workerStatus, + status: materializedWorkerStatus, }); } const seeded = await postJson( "/agent/test/agent-org/stale-workers/seed-run", { - org_id: `e2e-delete-${label}-${RUN_ID}`, + org_id: orgId, coordinator_agent_id: "builtin:sde", root_session_id: rootSessionId, - root_status: rootStatus, - run_status: runStatus, + root_status: runStatus === "completed" ? "paused" : rootStatus, + run_status: runStatus === "completed" ? "paused" : runStatus, workers, } ); + if (runStatus === "completed") { + await postJson( + "/agent/test/agent-org/session-delete/fixture/seed-related", + { + root_session_id: rootSessionId, + root_final_status: rootStatus, + runs: workers.slice(0, 1).map((worker) => ({ + org_run_id: seeded.org_run_id, + worker_session_id: worker.session_id, + member_id: worker.member_id, + final_status: "completed", + })), + } + ); + } return { runId: seeded.org_run_id, + orgId, rootSessionId, workerSessionIds: workers.map((worker) => worker.session_id), }; } -async function refreshAndWaitForSidebarRow(sessionId) { +async function refreshAndWaitForSidebarRow(sessionId, agentOrgId) { unwrap( await invokeE2E("seedSidebarSession", { sessionId, name: `Agent Org delete ${sessionId}`, status: "completed", + agentOrgId, }), `seedSidebarSession(${sessionId})` ); @@ -122,6 +143,10 @@ async function chooseDeleteFromRenderedSidebarMenu(sessionId) { throw new Error(`native sidebar menu did not open for ${sessionId}`); } + // `aria-pressed` flips when React requests the native menu; AppKit may need + // one more run-loop turn before it can receive System Events keystrokes. + await browser.pause(300); + // WebDriver key actions target the WebView rather than the macOS menu // process. Native menus support type-to-select, so select the uniquely // named Delete item and confirm it with real OS key events. @@ -141,7 +166,7 @@ async function persistenceSnapshot(sessionIds, runIds) { } async function deleteHierarchyAndAssertGone(hierarchy) { - await refreshAndWaitForSidebarRow(hierarchy.rootSessionId); + await refreshAndWaitForSidebarRow(hierarchy.rootSessionId, hierarchy.orgId); await openRenderedSidebarSession(hierarchy.rootSessionId); await chooseDeleteFromRenderedSidebarMenu(hierarchy.rootSessionId); @@ -166,13 +191,13 @@ async function deleteHierarchyAndAssertGone(hierarchy) { hierarchy.rootSessionId, ...hierarchy.workerSessionIds, ]) { - if (snapshot.sessions[sessionId] !== false) { + if (snapshot.sessions[sessionId] !== null) { throw new Error( `deleted Rust session remained durable: ${sessionId} ${JSON.stringify(snapshot)}` ); } } - if (snapshot.runs[hierarchy.runId] !== false) { + if (snapshot.runs[hierarchy.runId] !== null) { throw new Error( `deleted run remained durable: ${JSON.stringify(snapshot)}` ); @@ -208,8 +233,10 @@ describe("Agent Org Rust session hierarchy deletion rendered UI", () => { [unrelated.runId] ); if ( - snapshot.sessions[unrelated.rootSessionId] !== true || - snapshot.runs[unrelated.runId] !== true + typeof snapshot.sessions[unrelated.rootSessionId] !== "object" || + snapshot.sessions[unrelated.rootSessionId] === null || + typeof snapshot.runs[unrelated.runId] !== "object" || + snapshot.runs[unrelated.runId] === null ) { throw new Error( `unrelated Agent Org was modified: ${JSON.stringify(snapshot)}`