diff --git a/crates/flowproof-agent/src/author.rs b/crates/flowproof-agent/src/author.rs index 5cefcea..970790e 100644 --- a/crates/flowproof-agent/src/author.rs +++ b/crates/flowproof-agent/src/author.rs @@ -483,6 +483,7 @@ fn action_targets(action: &ResolvedAction) -> Vec<&Target> { | ResolvedAction::Scroll { target: None, .. } | ResolvedAction::AssertSql { .. } | ResolvedAction::AssertApi { .. } + | ResolvedAction::AssertSpreadsheet { .. } | ResolvedAction::CaptureDownload { .. } => vec![], } } diff --git a/crates/flowproof-agent/src/recorder.rs b/crates/flowproof-agent/src/recorder.rs index cd34e69..434ca12 100644 --- a/crates/flowproof-agent/src/recorder.rs +++ b/crates/flowproof-agent/src/recorder.rs @@ -1008,6 +1008,36 @@ fn step_for(id: usize, intent: &str, app: &str, action: &ResolvedAction) -> Step }), ) } + ResolvedAction::AssertSpreadsheet { + path, + sheet, + at, + column, + row_contains, + equals, + contains, + timeout_ms, + } => { + let mut expect = serde_json::Map::new(); + if let Some(value) = equals { + expect.insert("equals".into(), value.as_str().into()); + } + if let Some(needle) = contains { + expect.insert("contains".into(), needle.as_str().into()); + } + expect.insert("timeout_ms".into(), (*timeout_ms).into()); + ( + Vec::new(), + Action::Assert(Assertion::Spreadsheet { + path: path.clone(), + sheet: sheet.clone(), + at: at.clone(), + column: column.clone(), + row_contains: row_contains.clone(), + expect: Some(serde_json::Value::Object(expect)), + }), + ) + } }; if app == "vision" { stamp_vision_relation(&mut selectors, action); @@ -1150,6 +1180,7 @@ fn action_selector(action: &ResolvedAction) -> Option { | ResolvedAction::Reload | ResolvedAction::AssertSql { .. } | ResolvedAction::AssertApi { .. } + | ResolvedAction::AssertSpreadsheet { .. } | ResolvedAction::AssertScreenshot { .. } | ResolvedAction::CaptureDownload { .. } => return None, }; @@ -1705,6 +1736,7 @@ impl ReuseCursor { | ResolvedAction::AssertCaptured { .. } | ResolvedAction::AssertSql { .. } | ResolvedAction::AssertApi { .. } + | ResolvedAction::AssertSpreadsheet { .. } ); if !is_assert { if let Some(selector) = action_selector(&action) { @@ -3466,6 +3498,38 @@ pub fn record_with_reuse_and_options( } } } + ResolvedAction::AssertSpreadsheet { + path, + sheet, + at, + column, + row_contains, + equals, + contains, + timeout_ms, + } => { + // The trace stores references, never values: a + // `${captured.x}` (the downloaded export's own path is + // the common case) resolves from this run's captures, + // then a `${VAR}` from the environment — the same + // two-step ladder `TypeText` resolves with. + let substituted = flowproof_trace::captures::substitute(path, &captures) + .map_err(|reason| RecordError::AssertMismatch { + intent: spec_step.intent().to_string(), + expected: "a remembered capture in the spreadsheet path".to_string(), + actual: reason, + })?; + let probe = flowproof_driver::oob::OobProbe::Spreadsheet { + path: flowproof_trace::secret::resolve_refs(&substituted)?, + sheet: sheet.clone(), + at: at.clone(), + column: column.clone(), + row_contains: row_contains.clone(), + equals: equals.clone(), + contains: contains.clone(), + }; + poll_oob(&probe, *timeout_ms, &spec_step.intent())?; + } ResolvedAction::PressKey { key, modifiers } => { let mods: Vec = modifiers.iter().map(driver_key_mod).collect(); diff --git a/crates/flowproof-agent/src/rules.rs b/crates/flowproof-agent/src/rules.rs index abb4528..78562a4 100644 --- a/crates/flowproof-agent/src/rules.rs +++ b/crates/flowproof-agent/src/rules.rs @@ -370,6 +370,17 @@ pub enum ResolvedAction { retry: Option, timeout_ms: u64, }, + /// Out-of-band spreadsheet assertion: the exported file on disk. + AssertSpreadsheet { + path: String, + sheet: Option, + at: Option, + column: Option, + row_contains: Option, + equals: Option, + contains: Option, + timeout_ms: u64, + }, } /// How an [`ResolvedAction::AssertText`] expectation compares against the @@ -1006,6 +1017,54 @@ fn resolve_step_inner(app: &str, step: &SpecStep) -> Result, .map_or(ASSERT_TIMEOUT_MS, |s| s * 1000), }]); } + SpecStep::AssertSpreadsheet { assert_spreadsheet } => { + // Exactly one cell address: `at` alone, or `column` and + // `row_contains` together — mirrors `assert_api`'s cross-field + // rules (a config mistake fails here, before anything records, + // not with a probe-time surprise). + let has_at = assert_spreadsheet.at.is_some(); + let has_anchor = + assert_spreadsheet.column.is_some() || assert_spreadsheet.row_contains.is_some(); + if has_at && has_anchor { + return Err(unresolvable( + &assert_spreadsheet.path, + "assert_spreadsheet sets `at` OR `column`+`row_contains`, not both", + )); + } + if !has_at && !has_anchor { + return Err(unresolvable( + &assert_spreadsheet.path, + "assert_spreadsheet needs either `at` or both `column` and `row_contains`", + )); + } + if has_anchor + && (assert_spreadsheet.column.is_none() + || assert_spreadsheet.row_contains.is_none()) + { + return Err(unresolvable( + &assert_spreadsheet.path, + "assert_spreadsheet's `column` and `row_contains` must be set together", + )); + } + if assert_spreadsheet.equals.is_some() && assert_spreadsheet.contains.is_some() { + return Err(unresolvable( + &assert_spreadsheet.path, + "assert_spreadsheet sets at most one of equals/contains", + )); + } + return Ok(vec![ResolvedAction::AssertSpreadsheet { + path: assert_spreadsheet.path.clone(), + sheet: assert_spreadsheet.sheet.clone(), + at: assert_spreadsheet.at.clone(), + column: assert_spreadsheet.column.clone(), + row_contains: assert_spreadsheet.row_contains.clone(), + equals: assert_spreadsheet.equals.clone(), + contains: assert_spreadsheet.contains.clone(), + timeout_ms: assert_spreadsheet + .timeout_seconds + .map_or(ASSERT_TIMEOUT_MS, |s| s * 1000), + }]); + } // A whole-run control assertion, not a per-step driver action: the // record-time store-guard and the replay scan enforce it out of band // against the captured corpus. It performs nothing on the UI, so it @@ -6642,6 +6701,135 @@ mod dialog_suffix_tests { } } +#[cfg(test)] +mod spreadsheet_assertion_tests { + use super::*; + + #[test] + fn assert_spreadsheet_by_at_threads_through() { + let spec: crate::spec::SpreadsheetAssertSpec = serde_yaml::from_str( + "path: ${captured.pir_export}\nsheet: Sheet1\nat: B2\nequals: \"12.50\"\n", + ) + .expect("spec parses"); + let step = SpecStep::AssertSpreadsheet { + assert_spreadsheet: spec, + }; + let actions = resolve_step("web", &step).expect("resolves"); + let ResolvedAction::AssertSpreadsheet { + path, + sheet, + at, + column, + row_contains, + equals, + .. + } = &actions[0] + else { + panic!("expected AssertSpreadsheet"); + }; + // A raw ref passes through untouched — resolution is probe-time only. + assert_eq!(path, "${captured.pir_export}"); + assert_eq!(sheet.as_deref(), Some("Sheet1")); + assert_eq!(at.as_deref(), Some("B2")); + assert_eq!(column, &None); + assert_eq!(row_contains, &None); + assert_eq!(equals.as_deref(), Some("12.50")); + } + + #[test] + fn assert_spreadsheet_by_column_and_row_contains_threads_through() { + let spec: crate::spec::SpreadsheetAssertSpec = serde_yaml::from_str( + "path: pir.xlsx\ncolumn: Net Price\nrow_contains: \"100-100\"\ncontains: \"12\"\n", + ) + .expect("spec parses"); + let actions = resolve_step( + "web", + &SpecStep::AssertSpreadsheet { + assert_spreadsheet: spec, + }, + ) + .expect("resolves"); + let ResolvedAction::AssertSpreadsheet { + at, + column, + row_contains, + contains, + .. + } = &actions[0] + else { + panic!("expected AssertSpreadsheet"); + }; + assert_eq!(at, &None); + assert_eq!(column.as_deref(), Some("Net Price")); + assert_eq!(row_contains.as_deref(), Some("100-100")); + assert_eq!(contains.as_deref(), Some("12")); + } + + #[test] + fn assert_spreadsheet_neither_at_nor_anchor_is_a_spec_error() { + let spec: crate::spec::SpreadsheetAssertSpec = + serde_yaml::from_str("path: pir.xlsx\n").expect("parses"); + let err = resolve_step( + "web", + &SpecStep::AssertSpreadsheet { + assert_spreadsheet: spec, + }, + ) + .expect_err("no cell address at all must fail early"); + assert!( + err.to_string().contains("`at`") && err.to_string().contains("row_contains"), + "names both forms: {err}" + ); + } + + #[test] + fn assert_spreadsheet_both_at_and_anchor_is_a_spec_error() { + let spec: crate::spec::SpreadsheetAssertSpec = + serde_yaml::from_str("path: pir.xlsx\nat: B2\ncolumn: Net Price\nrow_contains: x\n") + .expect("parses"); + let err = resolve_step( + "web", + &SpecStep::AssertSpreadsheet { + assert_spreadsheet: spec, + }, + ) + .expect_err("at and column/row_contains together is a contradiction"); + assert!(err.to_string().contains("not both"), "{err}"); + } + + #[test] + fn assert_spreadsheet_column_without_row_contains_is_a_spec_error() { + let spec: crate::spec::SpreadsheetAssertSpec = + serde_yaml::from_str("path: pir.xlsx\ncolumn: Net Price\n").expect("parses"); + let err = resolve_step( + "web", + &SpecStep::AssertSpreadsheet { + assert_spreadsheet: spec, + }, + ) + .expect_err("a header alone cannot name a row"); + assert!(err.to_string().contains("together"), "{err}"); + } + + #[test] + fn assert_spreadsheet_both_equals_and_contains_is_a_spec_error() { + let spec: crate::spec::SpreadsheetAssertSpec = + serde_yaml::from_str("path: pir.xlsx\nat: A1\nequals: x\ncontains: y\n") + .expect("parses"); + let err = resolve_step( + "web", + &SpecStep::AssertSpreadsheet { + assert_spreadsheet: spec, + }, + ) + .expect_err("one question, two spellings"); + assert!( + err.to_string().contains("at most one of equals/contains"), + "{err}" + ); + } +} + #[cfg(test)] mod capture_download_tests { use super::*; diff --git a/crates/flowproof-agent/src/spec.rs b/crates/flowproof-agent/src/spec.rs index 4532db2..c6984f3 100644 --- a/crates/flowproof-agent/src/spec.rs +++ b/crates/flowproof-agent/src/spec.rs @@ -578,6 +578,7 @@ fn validate_drags_are_asserted(steps: &[SpecStep]) -> Result<(), SpecError> { Some(SpecStep::Assert { .. }) | Some(SpecStep::AssertSql { .. }) | Some(SpecStep::AssertApi { .. }) + | Some(SpecStep::AssertSpreadsheet { .. }) | Some(SpecStep::AssertScreenshot { .. }) ); if !asserted { @@ -1440,6 +1441,9 @@ pub enum SpecStep { AssertApi { assert_api: ApiAssertSpec, }, + AssertSpreadsheet { + assert_spreadsheet: SpreadsheetAssertSpec, + }, AssertScreenshot { assert_screenshot: ScreenshotAssertSpec, }, @@ -1567,7 +1571,8 @@ fn parse_secret_selector(selector: &str) -> Result { impl SpecStep { const FORMS: &'static str = "a plain string, `rules: `, `assert: `, \ - `assert_sql: {...}`, `assert_api: {...}`, `assert_screenshot: {...}`, \ + `assert_sql: {...}`, `assert_api: {...}`, `assert_spreadsheet: {...}`, \ + `assert_screenshot: {...}`, \ `prompt: `, `assert_tool_call: `, \ `assert_no_tool_call: `, `assert_no_egress`, \ `assert_no_secret_leak: ${VAR}`, `repeat: {...}`, `when: ` with `steps:`, \ @@ -1686,6 +1691,11 @@ impl SpecStep { Some("assert_api") => serde_yaml::from_value(inner) .map(|assert_api| SpecStep::AssertApi { assert_api }) .map_err(|e| format!("in `assert_api` step: {e}")), + Some("assert_spreadsheet") => serde_yaml::from_value(inner) + .map(|assert_spreadsheet| SpecStep::AssertSpreadsheet { + assert_spreadsheet, + }) + .map_err(|e| format!("in `assert_spreadsheet` step: {e}")), Some("assert_screenshot") => serde_yaml::from_value(inner) .map(|assert_screenshot| SpecStep::AssertScreenshot { assert_screenshot }) .map_err(|e| format!("in `assert_screenshot` step: {e}")), @@ -1799,6 +1809,9 @@ impl Serialize for SpecStep { } => single(serializer, "assert_no_tool_call", assert_no_tool_call), SpecStep::AssertSql { assert_sql } => single(serializer, "assert_sql", assert_sql), SpecStep::AssertApi { assert_api } => single(serializer, "assert_api", assert_api), + SpecStep::AssertSpreadsheet { assert_spreadsheet } => { + single(serializer, "assert_spreadsheet", assert_spreadsheet) + } SpecStep::AssertScreenshot { assert_screenshot } => { single(serializer, "assert_screenshot", assert_screenshot) } @@ -1938,6 +1951,41 @@ pub struct ApiAssertSpec { pub retry: Option, } +/// ```yaml +/// - assert_spreadsheet: +/// path: ${captured.pir_export} # may carry ${captured.x} / ${VAR} +/// sheet: Sheet1 # optional; first sheet if absent +/// at: B2 # OR column + row_contains, not both +/// column: Net Price # header text (first row) +/// row_contains: "100-100" # anchor: unique row containing it +/// equals: "12.50" # optional; cell text, exact +/// ``` +/// Reads the file directly (`calamine`) — not through UI Automation over +/// Excel's grid, which is untested and known-flaky. The cell is addressed +/// EITHER by `at` alone OR by `column`+`row_contains` together; mixing the +/// two, or setting neither, is a parse-time error (see `resolve_step_inner`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SpreadsheetAssertSpec { + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sheet: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub column: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub row_contains: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub equals: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub contains: Option, + /// Auto-wait bound override (default 10s) — a just-finished download + /// may still be mid-write when the first poll fires. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_seconds: Option, +} + impl SpecStep { /// The deterministic action text when this step explicitly opted into /// rule authoring. `None` means routing was not forced by a per-step @@ -1965,6 +2013,9 @@ impl SpecStep { format!("sql {}: {}", assert_sql.connection, assert_sql.query) } SpecStep::AssertApi { assert_api } => format!("api {}", assert_api.request), + SpecStep::AssertSpreadsheet { assert_spreadsheet } => { + format!("spreadsheet {}", assert_spreadsheet.path) + } SpecStep::AssertScreenshot { assert_screenshot } => { format!("screenshot matches {}", assert_screenshot.name) } @@ -2614,6 +2665,41 @@ steps: assert!(msg.contains("assert_api"), "names the step kind: {msg}"); } + #[test] + fn typoed_assert_spreadsheet_field_error_names_field_and_step_kind() { + let err = FlowSpec::parse( + "name: x\napp: web\nsteps:\n - assert_spreadsheet:\n path: out.xlsx\n \ + at: B2\n equalz: '1'\n", + ) + .expect_err("typo'd inner field must fail"); + let msg = err.to_string(); + assert!(msg.contains("equalz"), "names the field: {msg}"); + assert!( + msg.contains("assert_spreadsheet"), + "names the step kind: {msg}" + ); + } + + #[test] + fn assert_spreadsheet_round_trips_through_yaml() { + let spec = FlowSpec::parse( + "name: x\napp: web\nsteps:\n - assert_spreadsheet:\n path: out.xlsx\n \ + column: Net Price\n row_contains: \"100-100\"\n equals: \"12.50\"\n", + ) + .expect("parses"); + let SpecStep::AssertSpreadsheet { + assert_spreadsheet, .. + } = &spec.steps[0] + else { + panic!("expected AssertSpreadsheet"); + }; + assert_eq!(assert_spreadsheet.path, "out.xlsx"); + assert_eq!(assert_spreadsheet.column.as_deref(), Some("Net Price")); + let text = serde_yaml::to_string(&spec.steps[0]).expect("serializes"); + let reparsed: SpecStep = serde_yaml::from_str(&text).expect("reparses"); + assert_eq!(reparsed, spec.steps[0]); + } + #[test] fn unknown_step_key_error_names_key_and_lists_forms() { let err = FlowSpec::parse("name: x\napp: web\nsteps:\n - assert_apy:\n request: x\n") diff --git a/crates/flowproof-driver/src/surface.rs b/crates/flowproof-driver/src/surface.rs index 3da12b2..619385b 100644 --- a/crates/flowproof-driver/src/surface.rs +++ b/crates/flowproof-driver/src/surface.rs @@ -208,6 +208,7 @@ impl AppDriver for SurfaceRegistry { fn occluding_element(&mut self, selector: &UiaSelector) -> Result, DriverError>; fn stage_mocks(&mut self, rules: Vec) -> Result<(), DriverError>; fn set_files(&mut self, selector: &UiaSelector, paths: &[String]) -> Result<(), DriverError>; + fn wait_for_download(&mut self, timeout: Duration) -> Result; fn context_click(&mut self, selector: &UiaSelector) -> Result<(), DriverError>; fn double_click(&mut self, selector: &UiaSelector) -> Result<(), DriverError>; fn hover(&mut self, selector: &UiaSelector) -> Result<(), DriverError>; diff --git a/crates/flowproof-replay/src/lib.rs b/crates/flowproof-replay/src/lib.rs index 114132e..bddaa1b 100644 --- a/crates/flowproof-replay/src/lib.rs +++ b/crates/flowproof-replay/src/lib.rs @@ -1551,6 +1551,43 @@ fn check_assertion( } Ok((verdict, rung)) } + Assertion::Spreadsheet { + path, + sheet, + at, + column, + row_contains, + expect, + } => { + // The trace stores references, never values: `${captured.x}` + // (the export this checks is often itself a captured download + // path) resolves from this run's captures, then `${VAR}` from + // the environment — the same two-step ladder `TypeText` uses. + let substituted = match flowproof_trace::captures::substitute(path, captures) { + Ok(text) => text, + Err(reason) => return Ok((Err(reason), None)), + }; + let path = flowproof_trace::secret::resolve_refs(&substituted)?; + let probe = flowproof_driver::oob::OobProbe::Spreadsheet { + path, + sheet: sheet.clone(), + at: at.clone(), + column: column.clone(), + row_contains: row_contains.clone(), + equals: expect + .as_ref() + .and_then(|e| e.get("equals")) + .and_then(|v| v.as_str()) + .map(str::to_string), + contains: expect + .as_ref() + .and_then(|e| e.get("contains")) + .and_then(|v| v.as_str()) + .map(str::to_string), + }; + let (verdict, rung, _) = poll_oob(&probe, oob_timeout(expect.as_ref()))?; + Ok((verdict, rung)) + } other => Ok(( Err(format!( "assertion kind not supported in this slice: {other:?}"