diff --git a/Cargo.toml b/Cargo.toml index b7d66737..1db92686 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,6 +165,16 @@ tempfile = "3.0" chrono = { version = "0.4.44", default-features = false, features = ["clock", "serde"] } chrono-english = "0.1.8" rust_decimal = { version = "1.37", default-features = false, features = ["std", "serde", "serde-with-str"] } +polars = { version = "0.46", default-features = false, features = [ + "lazy", + "dtype-struct", + "dtype-decimal", + "strings", + "temporal", + "is_in", + "abs", + "regex", +] } toon = "0.1" # LLM / REPL diff --git a/apis/architect-exchange/domain.yaml b/apis/architect-exchange/domain.yaml index 433bb561..648769e7 100644 --- a/apis/architect-exchange/domain.yaml +++ b/apis/architect-exchange/domain.yaml @@ -277,7 +277,7 @@ entities: id_from: - account_id - symbol - description: Signed derivative exposure for one instrument in an account. + description: Signed derivative exposure for one instrument in an account. Ad-hoc ranking and money sums of fetched positions use program postfix, not extra catalog entities. discovery: names: - position @@ -940,7 +940,7 @@ entities: Fill: id_field: trade_id - description: Private execution against an account, including fees and side. + description: Private execution against an account, including fees and side. Ad-hoc rollups (fees by symbol, notional) use program postfix on fetched rows, not extra catalog entities. discovery: names: - fill diff --git a/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs b/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs index 5b5f9f8b..967b1986 100644 --- a/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs +++ b/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs @@ -13,7 +13,9 @@ use crate::execute_session::ExecuteSession; use crate::server_state::PlasmHostState; use super::ctx::GraphSurfaceWalkCtx; -use super::walk::{collect_entities, collect_row_json, snapshot_hot_entities, stream_rows}; +#[cfg(test)] +use super::walk::stream_rows; +use super::walk::{collect_entities, collect_row_json, snapshot_hot_entities}; /// Hot-cache snapshot + target count for spill rehydrate after the graph lock is released. pub(crate) struct GraphSpillSyncPlan { @@ -233,6 +235,7 @@ impl<'a> GraphSurfaceRehydrator<'a> { self.rehydrate_rows(hot, entity_type, logical_count).await } + #[cfg(test)] pub(crate) async fn stream_entity_rows( &self, hot_snapshot: Arc<[CachedEntity]>, diff --git a/crates/plasm-agent-core/src/graph_rehydrate/walk.rs b/crates/plasm-agent-core/src/graph_rehydrate/walk.rs index de927625..004f8e29 100644 --- a/crates/plasm-agent-core/src/graph_rehydrate/walk.rs +++ b/crates/plasm-agent-core/src/graph_rehydrate/walk.rs @@ -125,6 +125,7 @@ where }) } +#[cfg(test)] pub(crate) async fn stream_rows( ctx: &GraphSurfaceWalkCtx<'_>, hot_snapshot: Arc<[CachedEntity]>, @@ -180,7 +181,7 @@ pub(crate) async fn collect_entities( out.truncate(logical_count); crate::graph_cache_metrics::record_graph_rehydrate( "full", - out.len(), + stats.rows_yielded, stats.pages_read, started.elapsed(), ); diff --git a/crates/plasm-agent-core/src/plan_dry_display.rs b/crates/plasm-agent-core/src/plan_dry_display.rs index b4d51d1a..ae2d85b3 100644 --- a/crates/plasm-agent-core/src/plan_dry_display.rs +++ b/crates/plasm-agent-core/src/plan_dry_display.rs @@ -156,6 +156,9 @@ pub enum PlanDryOp { Dedupe { keys: Vec, }, + With { + columns: Vec, + }, Render { columns: Vec, template_chars: usize, @@ -336,6 +339,7 @@ pub(crate) fn human_ux_headline_for_op(op: &PlanDryOp) -> String { PlanDryOp::Limit { count } => format!("Take first {count}"), PlanDryOp::Dedupe { keys } if keys.is_empty() => "Distinct rows".into(), PlanDryOp::Dedupe { keys } => format!("Dedupe on {}", keys.join(", ")), + PlanDryOp::With { columns } => format!("Add columns {}", columns.join(", ")), PlanDryOp::Render { .. } => "Render text".into(), PlanDryOp::ForEach { .. } => "For each row".into(), PlanDryOp::Relation { .. } => "Follow relation".into(), @@ -374,6 +378,7 @@ pub(crate) fn human_ux_summary_for_op(op: &PlanDryOp) -> String { PlanDryOp::Aggregate { .. } => "Summarize".into(), PlanDryOp::Dedupe { keys } if keys.is_empty() => "Distinct rows".into(), PlanDryOp::Dedupe { keys } => format!("Dedupe on {}", keys.join(", ")), + PlanDryOp::With { columns } => format!("Add {}", columns.join(", ")), PlanDryOp::Render { columns, .. } => format!("Render {}", columns.join(", ")), PlanDryOp::Relation { relation, target, .. @@ -408,6 +413,7 @@ pub(crate) fn render_plan_dry_op(op: &PlanDryOp) -> String { format!("dedupe {}", keys.join(", ")) } } + PlanDryOp::With { columns } => format!("with {}", columns.join(", ")), PlanDryOp::Render { columns, template_chars, @@ -499,6 +505,12 @@ fn compact_op_from_compute( ComputeOp::DedupeBy { keys } => PlanDryOp::Dedupe { keys: keys.iter().map(|k| k.dotted()).collect(), }, + ComputeOp::With { columns } => PlanDryOp::With { + columns: columns + .iter() + .map(|c| c.name.as_str().to_string()) + .collect(), + }, ComputeOp::Render { columns, template, .. } => PlanDryOp::Render { diff --git a/crates/plasm-agent-core/src/plan_flow.rs b/crates/plasm-agent-core/src/plan_flow.rs index be9a45c4..ace116b9 100644 --- a/crates/plasm-agent-core/src/plan_flow.rs +++ b/crates/plasm-agent-core/src/plan_flow.rs @@ -624,7 +624,8 @@ impl<'a, P: FlowPolicyEvaluator + ?Sized> FlowPass<'a, P> { ComputeOp::Filter { .. } | ComputeOp::Sort { .. } | ComputeOp::Limit { .. } - | ComputeOp::DedupeBy { .. } => { + | ComputeOp::DedupeBy { .. } + | ComputeOp::With { .. } => { out = source_facts.clone(); } ComputeOp::GroupBy { aggregates, .. } | ComputeOp::Aggregate { aggregates, .. } => { diff --git a/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs b/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs index 2781854d..16cc206c 100644 --- a/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs +++ b/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs @@ -70,7 +70,26 @@ pub(in crate::plasm_dag) fn postfix_op_to_compute( cgs: cgs.as_ref(), symbol_map: None, }; - plasm_core::type_check_row_predicate(&row_pred, &tc_ctx).map_err(|e| e.to_string())?; + let predicates = crate::row_predicate_lower::lower_row_predicate_to_plan( + &row_pred, + session, + &qe, + state.cross_cache, + )?; + let extra = resolve_immediate_compute_schema(state, staged, source); + let mut catalog_pred = row_pred.clone(); + if let Some(schema) = extra.as_ref() { + catalog_pred.0.retain(|c| { + !schema + .fields + .iter() + .any(|f| f.name.as_str() == c.field.as_str()) + }); + } + if !catalog_pred.0.is_empty() { + plasm_core::type_check_row_predicate(&catalog_pred, &tc_ctx) + .map_err(|e| e.to_string())?; + } let mut paths = Vec::new(); for clause in &row_pred.0 { paths.push(FieldPath::from_dotted(clause.field.as_str())?); @@ -85,13 +104,13 @@ pub(in crate::plasm_dag) fn postfix_op_to_compute( "filter(...)", )?; } - let predicates = crate::row_predicate_lower::lower_row_predicate_to_plan( - &row_pred, + let schema = compute_passthrough_or_fallback_schema( session, - &qe, - state.cross_cache, - )?; - let schema = synthetic_schema_passthrough_rows(session, state, staged, source)?; + state, + staged, + source, + "PlanFilter", + ); Ok(mk(ComputeOp::Filter { predicates }, schema, false)) } PlasmPostfixOp::Sort { args } => { @@ -229,10 +248,43 @@ pub(in crate::plasm_dag) fn postfix_op_to_compute( let schema = synthetic_schema_passthrough_rows(session, state, staged, source)?; Ok(mk(ComputeOp::DedupeBy { keys: vec![] }, schema, false)) } + PlasmPostfixOp::With { body } => { + let columns = plasm_core::parse_with_body(body).map_err(|e| e.to_string())?; + let schema = synthetic_schema_passthrough_rows(session, state, staged, source)?; + let mut schema = schema; + for col in &columns { + schema.fields.push(plasm_core::SyntheticFieldSchema { + name: col.name.clone(), + value_kind: SyntheticValueKind::Unknown, + source: None, + }); + } + Ok(mk(ComputeOp::With { columns }, schema, false)) + } PlasmPostfixOp::Projection { fields } => { let qe = resolve_qualified_entity_for_dag_source(state, staged, source.to_string()); + let source_schema = resolve_immediate_compute_schema(state, staged, source); let mut map = BTreeMap::new(); - for field in parse_field_list(session, state.cross_cache, qe.as_ref(), fields)? { + for field in + parse_field_list(session, state.cross_cache, qe.as_ref(), fields).or_else(|_| { + fields + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|raw| { + let path = FieldPath::from_dotted(raw)?; + let resolved = resolve_sort_field_path( + session, + state.cross_cache, + qe.as_ref(), + source_schema.as_ref(), + &path, + )?; + Ok(resolved.dotted()) + }) + .collect::, String>>() + })? + { map.insert( OutputName::new(field.clone())?, FieldPath::from_dotted(&field)?, diff --git a/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs b/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs index 2caae3be..f37216f9 100644 --- a/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs +++ b/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs @@ -92,6 +92,7 @@ pub(in crate::plasm_dag) fn row_suffix_to_postfix(suffix: &RowSuffix) -> Option< RowSuffix::GroupBy { args } => Some(PlasmPostfixOp::GroupBy { args: args.clone() }), RowSuffix::Dedupe { keys } => Some(PlasmPostfixOp::Dedupe { keys: keys.clone() }), RowSuffix::Distinct { keys } => Some(PlasmPostfixOp::Distinct { keys: keys.clone() }), + RowSuffix::With { body } => Some(PlasmPostfixOp::With { body: body.clone() }), RowSuffix::Singleton => Some(PlasmPostfixOp::Singleton), RowSuffix::PageSize { n } => Some(PlasmPostfixOp::PageSize(*n as usize)), RowSuffix::Relation { .. } => None, diff --git a/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs b/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs index 69f3930d..8073fac9 100644 --- a/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs +++ b/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs @@ -36,21 +36,16 @@ pub(in crate::plasm_dag) fn infer_render_columns_for_node( cols.extend(aggregates.iter().map(|a| a.name.clone())); Ok(cols) } - ComputeOp::Sort { .. } | ComputeOp::Limit { .. } | ComputeOp::DedupeBy { .. } => { + ComputeOp::Sort { .. } | ComputeOp::Limit { .. } | ComputeOp::DedupeBy { .. } | ComputeOp::Filter { .. } => { let parent = lookup_dag_node(state, staged, parent_id.as_str()).ok_or_else(|| { format!("template column inference: missing upstream node `{parent_id}`") })?; infer_render_columns_for_node(session, state, staged, parent) } + ComputeOp::With { .. } => Ok(schema.fields.iter().map(|f| f.name.clone()).collect()), ComputeOp::Render { .. } => Err( "cannot infer columns from a row-to-text template result; bind a row-producing query/relation/projection, or write explicit `[field,...] < { - let parent = lookup_dag_node(state, staged, parent_id.as_str()).ok_or_else(|| { - format!("template column inference: missing upstream node `{parent_id}`") - })?; - infer_render_columns_for_node(session, state, staged, parent) - } }, DagNodeSource::Surface { qualified_entity, .. diff --git a/crates/plasm-agent-core/src/plasm_plan.rs b/crates/plasm-agent-core/src/plasm_plan.rs index f3948ab7..d4d95d3c 100644 --- a/crates/plasm-agent-core/src/plasm_plan.rs +++ b/crates/plasm-agent-core/src/plasm_plan.rs @@ -1660,7 +1660,8 @@ fn analyze_static_cardinality( ComputeOp::Project { .. } | ComputeOp::Filter { .. } | ComputeOp::Sort { .. } - | ComputeOp::DedupeBy { .. } => inner(plan, by_id, &compute.source, memo), + | ComputeOp::DedupeBy { .. } + | ComputeOp::With { .. } => inner(plan, by_id, &compute.source, memo), ComputeOp::Limit { count } if *count <= 1 => { CardinalityAnalysis::StaticSingleton } @@ -1755,7 +1756,8 @@ fn validated_analyze_static_cardinality( ComputeOp::Project { .. } | ComputeOp::Filter { .. } | ComputeOp::Sort { .. } - | ComputeOp::DedupeBy { .. } => inner(plan, by_id, c.compute.source.as_str(), memo), + | ComputeOp::DedupeBy { .. } + | ComputeOp::With { .. } => inner(plan, by_id, c.compute.source.as_str(), memo), ComputeOp::Limit { count } if *count <= 1 => CardinalityAnalysis::StaticSingleton, ComputeOp::Limit { .. } | ComputeOp::GroupBy { .. } => { CardinalityAnalysis::PluralOrUnknown @@ -1825,6 +1827,11 @@ fn validate_compute_template( validate_predicate(p, node_index, j)?; } } + ComputeOp::With { columns } if columns.is_empty() => { + return Err(format!( + "plan.nodes[{node_index}].compute.with.columns must be non-empty" + )); + } ComputeOp::GroupBy { aggregates, .. } | ComputeOp::Aggregate { aggregates } => { if aggregates.is_empty() { return Err(format!( diff --git a/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs b/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs index 7a6d27f3..d332f11b 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs @@ -2,11 +2,11 @@ use std::collections::BTreeMap; use std::sync::Arc; use minijinja::value::{Enumerator, Object, ObjectRepr}; +use plasm_runtime::{eval_compute_ops, ComputeEvalOutcome}; use crate::plasm_plan::OutputName; use crate::plasm_render_compile::render_context_hint; -use super::super::value_at_field_path as value_at_path; use super::super::*; pub(crate) async fn eval_compute_with_row_source( @@ -18,94 +18,15 @@ pub(crate) async fn eval_compute_with_row_source( session_id: &str, cgs: &CGS, ) -> Result, String> { - match row_source { - MaterializedRowSource::Inline(rows) => { - eval_compute_from_rows(compute, rows, cross_binding_rows) - } - MaterializedRowSource::GraphBacked { - entity_type, - logical_count, - hot_snapshot, - } => { - if matches!(&compute.op, ComputeOp::Render { .. }) { - let rows = - crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) - .resolve_row_source_rows( - row_source, - Some(crate::plasm_plan::PLAN_RENDER_MAX_ROWS), - ) - .await?; - return eval_compute_from_rows(compute, &rows, cross_binding_rows); - } - if compute_needs_full_materialize(&compute.op) { - let rows = - crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) - .rehydrate_rows( - std::sync::Arc::clone(hot_snapshot), - entity_type, - *logical_count, - ) - .await?; - return eval_compute_from_rows(compute, &rows, cross_binding_rows); - } - eval_compute_streaming( - compute, - es, - st, - session_id, - entity_type, - cgs, - std::sync::Arc::clone(hot_snapshot), - ) - .await - } - } -} - -pub(crate) async fn eval_compute_streaming( - compute: &ComputeTemplate, - es: &ExecuteSession, - st: &PlasmHostState, - session_id: &str, - entity_type: &str, - cgs: &CGS, - hot_snapshot: std::sync::Arc<[plasm_runtime::CachedEntity]>, -) -> Result, String> { - let mut out = Vec::new(); - let limit = match &compute.op { - ComputeOp::Limit { count } => Some(*count), - _ => None, + let cap = if matches!(&compute.op, ComputeOp::Render { .. }) { + Some(crate::plasm_plan::PLAN_RENDER_MAX_ROWS) + } else { + None }; - crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) - .stream_entity_rows(hot_snapshot, entity_type, |row| { - match &compute.op { - ComputeOp::Filter { predicates } => { - if predicates.iter().all(|p| predicate_matches(row, p)) { - out.push(row.clone()); - } - } - ComputeOp::Limit { .. } => out.push(row.clone()), - ComputeOp::Project { fields } => { - let mut obj = serde_json::Map::new(); - for (name, path) in fields { - obj.insert( - name.as_str().to_string(), - value_at_path(row, path) - .cloned() - .unwrap_or(serde_json::Value::Null), - ); - } - out.push(serde_json::Value::Object(obj)); - } - _ => {} - } - limit.is_some_and(|cap| out.len() >= cap) - }) + let rows = crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) + .resolve_row_source_rows(row_source, cap) .await?; - if let ComputeOp::Limit { count } = &compute.op { - out.truncate(*count); - } - Ok(out) + eval_compute_from_rows(compute, &rows, cross_binding_rows) } pub(crate) fn eval_compute_from_rows( @@ -113,194 +34,27 @@ pub(crate) fn eval_compute_from_rows( rows: &[serde_json::Value], cross_binding_rows: &BTreeMap>, ) -> Result, String> { - match &compute.op { - ComputeOp::Project { fields } => rows - .iter() - .map(|row| { - let mut out = serde_json::Map::new(); - for (name, path) in fields { - out.insert( - name.as_str().to_string(), - value_at_path(row, path) - .cloned() - .unwrap_or(serde_json::Value::Null), - ); - } - Ok(serde_json::Value::Object(out)) - }) - .collect(), - ComputeOp::Filter { predicates } => Ok(rows - .iter() - .filter(|row| predicates.iter().all(|p| predicate_matches(row, p))) - .cloned() - .collect()), - ComputeOp::GroupBy { keys, aggregates } => group_rows(rows, keys, aggregates), - ComputeOp::Aggregate { aggregates } => aggregate_rows(rows, aggregates), - ComputeOp::Sort { key, descending } => { - let mut sorted = rows.to_vec(); - sorted - .sort_by(|a, b| cmp_json_sort_values(value_at_path(a, key), value_at_path(b, key))); - if *descending { - sorted.reverse(); - } - Ok(sorted) - } - ComputeOp::Limit { count } => Ok(rows.iter().take(*count).cloned().collect()), - ComputeOp::DedupeBy { keys } => dedupe_rows(rows, keys), - ComputeOp::Render { + match eval_compute_ops(std::slice::from_ref(&compute.op), rows)? { + ComputeEvalOutcome::Rows(out) => Ok(out), + ComputeEvalOutcome::Render { + rows, columns, - template, column_aliases, - render_bindings, - } => render_compute(&RenderComputeInput { - primary_rows: rows, - columns: &RenderColumns::from_op_parts(columns.clone(), column_aliases.clone()), template, - collection_alias: compute.collection_alias.as_ref(), + collection_alias, render_bindings, + } => render_compute(&RenderComputeInput { + primary_rows: &rows, + columns: &RenderColumns::from_op_parts(columns, column_aliases), + template: &template, + collection_alias: collection_alias + .as_ref() + .or(compute.collection_alias.as_ref()), + render_bindings: &render_bindings, binding_rows: cross_binding_rows, }), } } -pub(crate) fn dedupe_rows( - rows: &[serde_json::Value], - keys: &[FieldPath], -) -> Result, String> { - use std::collections::HashSet; - let mut seen = HashSet::new(); - let mut out = Vec::new(); - for row in rows { - let composite = if keys.is_empty() { - serde_json::to_string(row).unwrap_or_default() - } else { - let parts: Vec = keys - .iter() - .map(|k| { - value_at_path(row, k) - .map(json_scalar_display) - .unwrap_or_default() - }) - .collect(); - serde_json::to_string(&parts).unwrap_or_default() - }; - if seen.insert(composite) { - out.push(row.clone()); - } - } - Ok(out) -} - -pub(crate) fn group_rows( - rows: &[serde_json::Value], - keys: &[FieldPath], - aggregates: &[crate::plasm_plan::AggregateSpec], -) -> Result, String> { - if keys.is_empty() { - return Err("group_by requires at least one key".into()); - } - let mut groups: BTreeMap> = BTreeMap::new(); - for row in rows { - let parts: Vec = keys - .iter() - .map(|k| { - value_at_path(row, k) - .map(json_scalar_display) - .unwrap_or_default() - }) - .collect(); - let composite = serde_json::to_string(&parts).unwrap_or_default(); - groups.entry(composite).or_default().push(row); - } - let mut out = Vec::new(); - for (composite, group_rows) in groups { - let parts: Vec = serde_json::from_str(&composite).unwrap_or_default(); - let mut obj = serde_json::Map::new(); - for (key_path, part) in keys.iter().zip(parts.iter()) { - obj.insert(key_path.dotted(), serde_json::Value::String(part.clone())); - } - append_aggregates(&mut obj, &group_rows, aggregates)?; - out.push(serde_json::Value::Object(obj)); - } - Ok(out) -} - -pub(crate) fn aggregate_rows( - rows: &[serde_json::Value], - aggregates: &[crate::plasm_plan::AggregateSpec], -) -> Result, String> { - let refs = rows.iter().collect::>(); - let mut obj = serde_json::Map::new(); - append_aggregates(&mut obj, &refs, aggregates)?; - Ok(vec![serde_json::Value::Object(obj)]) -} - -pub(crate) fn append_aggregates( - obj: &mut serde_json::Map, - rows: &[&serde_json::Value], - aggregates: &[crate::plasm_plan::AggregateSpec], -) -> Result<(), String> { - for agg in aggregates { - let value = match agg.function { - AggregateFunction::Count => serde_json::json!(rows.len()), - AggregateFunction::Sum => { - serde_json::json!(aggregate_numbers(rows, agg.field.as_ref()) - .iter() - .sum::()) - } - AggregateFunction::Avg => { - let nums = aggregate_numbers(rows, agg.field.as_ref()); - serde_json::json!(if nums.is_empty() { - 0.0 - } else { - nums.iter().sum::() / nums.len() as f64 - }) - } - AggregateFunction::Min => aggregate_numbers(rows, agg.field.as_ref()) - .into_iter() - .reduce(f64::min) - .map(|n| serde_json::json!(n)) - .unwrap_or(serde_json::Value::Null), - AggregateFunction::Max => aggregate_numbers(rows, agg.field.as_ref()) - .into_iter() - .reduce(f64::max) - .map(|n| serde_json::json!(n)) - .unwrap_or(serde_json::Value::Null), - AggregateFunction::First => rows - .first() - .and_then(|row| { - agg.field - .as_ref() - .and_then(|f| value_at_path(row, f)) - .cloned() - }) - .unwrap_or(serde_json::Value::Null), - AggregateFunction::Last => rows - .last() - .and_then(|row| { - agg.field - .as_ref() - .and_then(|f| value_at_path(row, f)) - .cloned() - }) - .unwrap_or(serde_json::Value::Null), - }; - obj.insert(agg.name.as_str().to_string(), value); - } - Ok(()) -} - -pub(crate) fn aggregate_numbers( - rows: &[&serde_json::Value], - field: Option<&FieldPath>, -) -> Vec { - rows.iter() - .filter_map(|row| { - field - .and_then(|f| value_at_path(row, f)) - .and_then(json_number) - }) - .collect() -} pub(crate) struct RenderComputeInput<'a> { pub primary_rows: &'a [serde_json::Value], @@ -404,7 +158,6 @@ impl Object for RenderBindingValue { } fn get_value(self: &Arc, key: &minijinja::Value) -> Option { - // Attribute access (`items.title`) delegates to the first row. if let Some(name) = key.as_str() { return self .rows @@ -412,7 +165,6 @@ impl Object for RenderBindingValue { .and_then(|row| row.get_attr(name).ok()) .filter(|value| !value.is_undefined()); } - // Sequence index access (`items[0]`) and `{% for … %}` iteration. usize::try_from(key.clone()) .ok() .and_then(|idx| self.rows.get(idx).cloned()) @@ -458,10 +210,6 @@ pub(crate) fn binding_rows_for_render( Ok(out) } -pub(crate) fn json_number(v: &serde_json::Value) -> Option { - v.as_f64().or_else(|| v.as_i64().map(|n| n as f64)) -} - pub(crate) fn json_scalar_display(v: &serde_json::Value) -> String { match v { serde_json::Value::String(s) => s.clone(), @@ -484,37 +232,6 @@ pub(crate) fn json_plasm_literal_display(v: &serde_json::Value) -> String { } } -pub(crate) fn sort_display_key(v: Option<&serde_json::Value>) -> String { - v.map(json_scalar_display).unwrap_or_default() -} - -/// Compare two JSON cell values for deterministic `.sort(...)` ordering. -/// -/// When both values are numeric (JSON numbers or strings that parse as integers/floats), ordering is -/// numeric so multi-digit values sort correctly (`87` before `300`). Otherwise ordering follows the -/// legacy string collation used by [`sort_display_key`] (including missing/`null` → empty string). -pub(crate) fn cmp_json_sort_values( - a: Option<&serde_json::Value>, - b: Option<&serde_json::Value>, -) -> std::cmp::Ordering { - match (a, b) { - (Some(va), Some(vb)) => { - if let (Some(na), Some(nb)) = (json_number(va), json_number(vb)) { - return na.total_cmp(&nb); - } - if let (Some(sa), Some(sb)) = (va.as_str(), vb.as_str()) { - if let (Ok(ia), Ok(ib)) = (sa.parse::(), sb.parse::()) { - return ia.cmp(&ib); - } - if let (Ok(fa), Ok(fb)) = (sa.parse::(), sb.parse::()) { - return fa.total_cmp(&fb); - } - } - sort_display_key(Some(va)).cmp(&sort_display_key(Some(vb))) - } - _ => sort_display_key(a).cmp(&sort_display_key(b)), - } -} pub(crate) fn compute_fingerprint(node: &ValidatedPlanNode, rows: &[serde_json::Value]) -> String { use sha2::{Digest, Sha256}; let mut hasher = Sha256::new(); diff --git a/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs b/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs index d2cc7503..13ec9a7e 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs @@ -173,6 +173,15 @@ pub(crate) fn render_compute_template(compute: &ComputeTemplate) -> String { ) } } + ComputeOp::With { columns } => format!( + "with {} [{}]", + compute.source, + columns + .iter() + .map(|c| c.name.as_str()) + .collect::>() + .join(", ") + ), ComputeOp::Render { columns, template, .. } => format!( diff --git a/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs b/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs index 0c8fd4ab..6cb96922 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs @@ -485,16 +485,6 @@ pub(crate) async fn materialized_rows( .await } -pub(crate) fn compute_needs_full_materialize(op: &ComputeOp) -> bool { - matches!( - op, - ComputeOp::Sort { .. } - | ComputeOp::GroupBy { .. } - | ComputeOp::Aggregate { .. } - | ComputeOp::DedupeBy { .. } - ) -} - #[must_use] pub(crate) fn execution_result_from_fanout_fold( fold: super::plan_fanout_parallel::PlanLineExecutionFold, diff --git a/crates/plasm-agent-core/src/plasm_plan_run/mod.rs b/crates/plasm-agent-core/src/plasm_plan_run/mod.rs index 58733923..7e81864d 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/mod.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/mod.rs @@ -27,8 +27,8 @@ use crate::http_execute::{ use crate::plan_dry_display; pub use crate::plan_dry_display::PlanDryReview; use crate::plasm_plan::{ - AggregateFunction, BindingName, ComputeOp, ComputeTemplate, EffectClass, FieldPath, InputAlias, - Plan, PlanExprTemplate, PlanNodeId, PlanNodeKind, PlanResultUse, PlanValue, QualifiedEntityKey, + BindingName, ComputeOp, ComputeTemplate, EffectClass, InputAlias, Plan, PlanExprTemplate, + PlanNodeId, PlanNodeKind, PlanResultUse, PlanValue, QualifiedEntityKey, RelationSourceCardinality, ValidatedForEachNode, ValidatedPlan, ValidatedPlanDataInput, ValidatedPlanExprTemplate, ValidatedPlanNode, ValidatedPlanState, ValidatedRelationTraversalNode, PLAN_RENDER_MAX_OUTPUT_CHARS, PLAN_RENDER_MAX_ROWS, @@ -102,8 +102,7 @@ pub(crate) use parse::{ entry_scoped_execute_session, propagate_row_identities, row_identities_from_entities, }; pub(crate) use row_json::{ - cached_entity_row_json, predicate_matches, value_at_dotted, value_at_field_path, - value_at_segments, + cached_entity_row_json, predicate_matches, value_at_dotted, value_at_segments, }; #[cfg(test)] diff --git a/crates/plasm-agent-core/src/plasm_plan_run/parse.rs b/crates/plasm-agent-core/src/plasm_plan_run/parse.rs index 166fb741..bc894d25 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/parse.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/parse.rs @@ -452,6 +452,7 @@ pub(crate) fn propagate_row_identities( match op { ComputeOp::Limit { count } => Ok(mat.row_identities.iter().take(*count).cloned().collect()), ComputeOp::Project { .. } => Ok(mat.row_identities.iter().take(out_len).cloned().collect()), + ComputeOp::With { .. } => Ok(mat.row_identities.iter().take(out_len).cloned().collect()), ComputeOp::Filter { predicates } => { let Some(rows) = mat.row_source.inline_rows() else { return Ok(Vec::new()); diff --git a/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs b/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs index 7b1b72fd..aa72a8da 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs @@ -1,7 +1,6 @@ //! Row JSON helpers. use super::*; -use crate::plasm_plan::FieldPath; pub(crate) fn cached_entity_row_json(entity: &CachedEntity, cgs: &CGS) -> serde_json::Value { entity_to_row_json(entity, Some(cgs)) @@ -18,13 +17,6 @@ pub(crate) fn value_at_segments<'a>( Some(cur) } -pub(crate) fn value_at_field_path<'a>( - row: &'a serde_json::Value, - path: &FieldPath, -) -> Option<&'a serde_json::Value> { - value_at_segments(row, path.segments()) -} - pub(crate) fn value_at_dotted<'a>( row: &'a serde_json::Value, path: &str, diff --git a/crates/plasm-agent-core/src/plasm_plan_run/step_materialize.rs b/crates/plasm-agent-core/src/plasm_plan_run/step_materialize.rs index 707e6a70..cc904de1 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/step_materialize.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/step_materialize.rs @@ -111,9 +111,9 @@ pub(crate) async fn materialize_executable_plan_step( /// Live materialization of a pure step through the shared [`PureStep::materialize`] kernel. /// /// `Compute` over a GraphBacked source is the one arm that cannot funnel its rows through the plain -/// kernel: live execute fuses the op with I/O streaming (bounded-RAM early-stop over spilled graph -/// pages). That fusion still evaluates the *same* `eval_compute_from_rows` op semantics — only row -/// *acquisition* differs — so it stays a pure step, just materialized against the live row source. +/// kernel: live execute first resolves the row source, then evaluates the shared +/// `eval_compute_from_rows` semantics against those rows. The op remains pure; only row acquisition +/// is host-backed. async fn live_materialize_pure( ctx: &PlanStepMaterializeCtx<'_>, pure: PureStep, diff --git a/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs b/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs index f1450a69..598b460f 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs @@ -6,23 +6,6 @@ use plasm_core::TeachingExposureSession; use std::path::PathBuf; use std::sync::Arc; -#[test] -fn cmp_json_sort_values_orders_multi_digit_numbers_numerically() { - use std::cmp::Ordering; - let n87 = serde_json::json!(87); - let n300 = serde_json::json!(300); - assert_eq!( - cmp_json_sort_values(Some(&n87), Some(&n300)), - Ordering::Less - ); - let s87 = serde_json::json!("87"); - let s300 = serde_json::json!("300"); - assert_eq!( - cmp_json_sort_values(Some(&s87), Some(&s300)), - Ordering::Less - ); -} - #[test] fn singleton_input_zero_row_error_is_actionable() { let err = singleton_input_row_count_error("src", "_", 0, "staged expression rendering"); @@ -39,40 +22,6 @@ fn singleton_input_multi_row_error_mentions_ambiguity_remedy() { assert!(err.contains(".singleton()"), "{err}"); } -#[test] -fn cmp_json_sort_values_string_collates_non_numeric_strings_lexically() { - use std::cmp::Ordering; - let apple = serde_json::json!("apple"); - let banana = serde_json::json!("banana"); - assert_eq!( - cmp_json_sort_values(Some(&apple), Some(&banana)), - Ordering::Less - ); -} - -/// Regression: `.sort(score)` must not stringify numbers and compare lexicographically (where -/// `87` sorts after `300`). Keeps parity with [`eval_compute`] `ComputeOp::Sort` staging. -#[test] -fn plan_sort_compute_orders_integer_scores_numerically() { - let key = FieldPath::from_dotted("score").expect("score path"); - let mut rows = [ - serde_json::json!({"id": "n300", "score": 300}), - serde_json::json!({"id": "n87", "score": 87}), - serde_json::json!({"id": "n100", "score": 100}), - ]; - rows.sort_by(|a, b| { - cmp_json_sort_values(value_at_field_path(a, &key), value_at_field_path(b, &key)) - }); - assert_eq!(rows[0]["id"], "n87"); - assert_eq!(rows[1]["id"], "n100"); - assert_eq!(rows[2]["id"], "n300"); - - rows.reverse(); - assert_eq!(rows[0]["id"], "n300"); - assert_eq!(rows[1]["id"], "n100"); - assert_eq!(rows[2]["id"], "n87"); -} - fn github_repository_commit_session() -> ExecuteSession { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let cgs = Arc::new(load_schema(&root.join("../../apis/github")).expect("load github")); diff --git a/crates/plasm-core/Cargo.toml b/crates/plasm-core/Cargo.toml index 0c867eed..8477b0e4 100644 --- a/crates/plasm-core/Cargo.toml +++ b/crates/plasm-core/Cargo.toml @@ -9,6 +9,8 @@ readme = "README.md" default = ["ranked_capability_gate"] # Non-read capabilities also require membership in an optional sorted capability-name list when that list is non-empty. ranked_capability_gate = [] +# Subscriber for the `dump_prompt` binary only — library code must not own a global subscriber. +dump-prompt = ["dep:tracing-subscriber"] [dependencies] bm25 = { workspace = true } @@ -22,7 +24,7 @@ chrono = { workspace = true } chrono-english = { workspace = true } rust_decimal = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["env-filter"] } +tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true } sha2 = { workspace = true } hex = { workspace = true } base64 = { workspace = true } @@ -30,6 +32,7 @@ rustc-hash = "2" riptoken = "0.3.0" rayon = { workspace = true } minijinja = { version = "2.19.0", default-features = false, features = ["builtins", "serde"] } + [dev-dependencies] insta = { workspace = true } proptest = { workspace = true } @@ -37,6 +40,12 @@ tempfile = { workspace = true } criterion = { workspace = true } plasm-discovery = { path = "../plasm-discovery" } plasm-discovery-eval = { path = "../plasm-discovery-eval" } +tracing-subscriber = { workspace = true, features = ["env-filter"] } + +[[bin]] +name = "dump_prompt" +path = "src/bin/dump_prompt.rs" +required-features = ["dump-prompt"] [[bench]] name = "schema_load" diff --git a/crates/plasm-core/src/expr_parser/postfix.rs b/crates/plasm-core/src/expr_parser/postfix.rs index f37d084e..3956cb10 100644 --- a/crates/plasm-core/src/expr_parser/postfix.rs +++ b/crates/plasm-core/src/expr_parser/postfix.rs @@ -22,6 +22,7 @@ pub enum PlasmPostfixOp { GroupBy { args: String }, Dedupe { keys: String }, Distinct { keys: Option }, + With { body: String }, Projection { fields: String }, } @@ -325,6 +326,14 @@ pub fn peel_postfix_suffixes(rhs: &str) -> Result<(String, Vec), }); cur = p; progressed = true; + } else if let Some((p, body)) = strip_trailing_brace_block(t, "with")? { + ops_rev.push(PlasmPostfixOp::With { body }); + cur = p; + progressed = true; + } else if let Some((p, body)) = strip_trailing_method_call(t, "with")? { + ops_rev.push(PlasmPostfixOp::With { body }); + cur = p; + progressed = true; } else if let Some((p, fields)) = strip_trailing_projection(t)? { ops_rev.push(PlasmPostfixOp::Projection { fields }); cur = p; @@ -684,6 +693,28 @@ mod tests { assert_eq!(ops3, vec![PlasmPostfixOp::Distinct { keys: None }]); } + #[test] + fn peel_with_brace_body() { + let (p, ops) = peel_postfix_suffixes("issues.with{age_days: (now - updated_at)}").unwrap(); + assert_eq!(p, "issues"); + assert_eq!( + ops, + vec![PlasmPostfixOp::With { + body: "age_days: (now - updated_at)".into() + }] + ); + } + + #[test] + fn peel_does_not_treat_join_or_open_as_row_compute() { + let (p, ops) = peel_postfix_suffixes("issues.join(comments)").unwrap(); + assert!(ops.is_empty(), "join is not a postfix verb, got {ops:?}"); + assert!(p.contains("join")); + let (p2, ops2) = peel_postfix_suffixes("issues.open(labels)").unwrap(); + assert!(ops2.is_empty(), "open is not a postfix verb, got {ops2:?}"); + assert!(p2.contains("open")); + } + #[test] fn render_tail_cross_binding_labels_before_heredoc() { let r = try_parse_render_tail("pika,repos < Result<(), String> { let sem_violations = cgs.string_semantics_violations(); if !sem_violations.is_empty() { for msg in &sem_violations { - error!(target: "plasm_core::cgs", "{}", msg); + error!(target: "plasm_core::cgs", violation = %msg, "string_semantics violation"); } return Err(format!( "CGS load requires string_semantics on every string field and string capability parameter ({} issue(s); first: {})", @@ -1048,7 +1048,7 @@ fn normalize_blob_field_type( /// without a `data_class` (plan-flow cannot label that data). fn warn_unlabeled_output_data(cgs: &CGS) { for msg in cgs.unlabeled_output_data_warnings() { - warn!(target: "plasm_core::loader", "{msg}"); + warn!(target: "plasm_core::loader", violation = %msg, "unlabeled output data"); } } diff --git a/crates/plasm-core/src/plasm_monad/mod.rs b/crates/plasm-core/src/plasm_monad/mod.rs index ed0d5d9e..8afb49db 100644 --- a/crates/plasm-core/src/plasm_monad/mod.rs +++ b/crates/plasm-core/src/plasm_monad/mod.rs @@ -21,12 +21,13 @@ pub use operators::{ plasm_parallel_return, plasm_pure_step, }; pub use payload::{ - AggregateFunction, AggregateSpec, BindingName, ComputeOp, ComputeTemplate, DeriveKind, + AggregateFunction, AggregateSpec, ArithOp, BindingName, ComputeOp, ComputeTemplate, DeriveKind, DerivePayload, DeriveTemplate, EffectTemplate, FieldPath, FlatMapEffectPayload, FlatMapRelationPayload, InputCardinality, InvokePayload, MapPayload, OutputName, PlanDataInput, PlanExprIr, PlanExprTemplate, PlanInputBinding, PlanPredicate, PlanPredicateOp, PlanQualifiedEntityKey, PlanRelationTraversal, PlanResultUse, PlasmDataValue, PlasmStepPayload, PurePayload, RelationCardinality, RelationName, RelationSourceCardinality, - SyntheticFieldSchema, SyntheticResultSchema, SyntheticValueKind, + SyntheticFieldSchema, SyntheticResultSchema, SyntheticValueKind, WithColumn, WithExpr, + WithExprError, WithLiteral, }; pub use step::{EffectBarrier, EffectClass, PlasmStep, PlasmStepKind, ResultShape, SurfaceKind}; diff --git a/crates/plasm-core/src/plasm_monad/payload/compute.rs b/crates/plasm-core/src/plasm_monad/payload/compute.rs index 3e33763c..fa4891d7 100644 --- a/crates/plasm-core/src/plasm_monad/payload/compute.rs +++ b/crates/plasm-core/src/plasm_monad/payload/compute.rs @@ -1,5 +1,6 @@ use super::atoms::{FieldPath, OutputName}; use super::value::PlanPredicate; +use super::with_expr::WithColumn; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -47,6 +48,9 @@ pub enum ComputeOp { #[serde(default, skip_serializing_if = "Vec::is_empty")] keys: Vec, }, + With { + columns: Vec, + }, Render { columns: Vec, template: String, @@ -109,6 +113,10 @@ pub enum SyntheticValueKind { String, Array, Object, + Money, + Temporal, + EntityRef, + Duration, Unknown, } diff --git a/crates/plasm-core/src/plasm_monad/payload/mod.rs b/crates/plasm-core/src/plasm_monad/payload/mod.rs index d88b6a04..1a52b871 100644 --- a/crates/plasm-core/src/plasm_monad/payload/mod.rs +++ b/crates/plasm-core/src/plasm_monad/payload/mod.rs @@ -5,6 +5,7 @@ mod relation; mod step_payload; mod templates; mod value; +mod with_expr; pub use crate::identity::RelationName; pub use atoms::{BindingName, FieldPath, OutputName, PlanQualifiedEntityKey}; @@ -23,3 +24,4 @@ pub use value::{ InputCardinality, PlanDataInput, PlanInputBinding, PlanPredicate, PlanPredicateOp, PlanResultUse, PlasmDataValue, }; +pub use with_expr::{ArithOp, WithColumn, WithExpr, WithExprError, WithLiteral}; diff --git a/crates/plasm-core/src/plasm_monad/payload/step_payload.rs b/crates/plasm-core/src/plasm_monad/payload/step_payload.rs index 39963513..7c1d8dbb 100644 --- a/crates/plasm-core/src/plasm_monad/payload/step_payload.rs +++ b/crates/plasm-core/src/plasm_monad/payload/step_payload.rs @@ -168,6 +168,7 @@ fn compute_op_label(op: &super::compute::ComputeOp) -> String { ComputeOp::Sort { .. } => "sort".into(), ComputeOp::Limit { count } => format!("limit {count}"), ComputeOp::DedupeBy { .. } => "dedupe_by".into(), + ComputeOp::With { .. } => "with".into(), ComputeOp::Render { .. } => "render".into(), } } diff --git a/crates/plasm-core/src/plasm_monad/payload/with_expr.rs b/crates/plasm-core/src/plasm_monad/payload/with_expr.rs new file mode 100644 index 00000000..a4b60460 --- /dev/null +++ b/crates/plasm-core/src/plasm_monad/payload/with_expr.rs @@ -0,0 +1,65 @@ +//! `.with` expression AST stored on hashed [`super::ComputeOp::With`]. + +use super::atoms::{FieldPath, OutputName}; +use super::value::PlanPredicateOp; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WithColumn { + pub name: OutputName, + pub expr: WithExpr, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WithExpr { + Field(FieldPath), + Literal(WithLiteral), + Arith { + op: ArithOp, + lhs: Box, + rhs: Box, + }, + /// Catalog-plane clock token (`now` → UTC). A catalog field named `now` loses. + Now, + Len { + field: FieldPath, + }, + When { + lhs: Box, + op: PlanPredicateOp, + rhs: Box, + then: Box, + else_: Box, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArithOp { + Add, + Sub, + Mul, + Div, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WithLiteral { + Null, + Bool(bool), + Integer(i64), + Number(String), + String(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum WithExprError { + #[error("empty .with body")] + EmptyBody, + #[error("invalid .with column `{0}`")] + BadColumn(String), + #[error("invalid .with expression: {0}")] + Parse(String), +} diff --git a/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt b/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt index b7b2dbe4..f4d84b30 100644 --- a/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt +++ b/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt @@ -41,7 +41,7 @@ TSV table semantics: Core surface: - Get identity: `e#(id)` (parens). Query/filter: `e#{field=…}` (braces). Search when taught: `e#~$` / `e#~"text"`. -- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `[field,…]`. +- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `.with{k: expr}` `[field,…]`. - Inline rows when delivery is `inline` (≤25); `snapshot_only` / `(in artifact)` need artifact read. Copy **`artifact_uri`** from the step — never plan `run_step` / `dict_ref`. - `page(...)` is HTTP-execute only — not an MCP tool argument. diff --git a/crates/plasm-core/src/row_composition.rs b/crates/plasm-core/src/row_composition.rs index 45babdc2..b4795fa2 100644 --- a/crates/plasm-core/src/row_composition.rs +++ b/crates/plasm-core/src/row_composition.rs @@ -71,6 +71,7 @@ pub enum RowSuffix { GroupBy { args: String }, Dedupe { keys: String }, Distinct { keys: Option }, + With { body: String }, Singleton, PageSize { n: u32 }, } @@ -92,6 +93,7 @@ impl RowSuffix { PlasmPostfixOp::GroupBy { args } => Ok(Self::GroupBy { args: args.clone() }), PlasmPostfixOp::Dedupe { keys } => Ok(Self::Dedupe { keys: keys.clone() }), PlasmPostfixOp::Distinct { keys } => Ok(Self::Distinct { keys: keys.clone() }), + PlasmPostfixOp::With { body } => Ok(Self::With { body: body.clone() }), PlasmPostfixOp::Singleton => Ok(Self::Singleton), PlasmPostfixOp::PageSize(n) => Ok(Self::PageSize { n: *n as u32 }), } diff --git a/crates/plasm-core/src/row_plan/collect.rs b/crates/plasm-core/src/row_plan/collect.rs new file mode 100644 index 00000000..4413bf2b --- /dev/null +++ b/crates/plasm-core/src/row_plan/collect.rs @@ -0,0 +1,64 @@ +//! Collect barriers — the only legal materialize points. + +use crate::plasm_monad::{OutputName, StepId}; +use serde::{Deserialize, Serialize}; +use std::num::NonZeroUsize; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectReason { + ProgramReturn { + step: StepId, + }, + PageContinue { + step: StepId, + page: PageCursor, + }, + InvokeArg { + consumer: StepId, + hole: String, + }, + Render { + step: StepId, + spec: RenderCollectSpec, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenderCollectSpec { + pub columns: Vec, + pub column_aliases: std::collections::BTreeMap, + pub template: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collection_alias: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub render_bindings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PageCursor { + pub token: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectCardinality { + List, + Single, + Page { size: PageSize }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct PageSize(NonZeroUsize); + +impl PageSize { + pub fn new(n: usize) -> Option { + NonZeroUsize::new(n).map(Self) + } + + #[must_use] + pub fn get(self) -> usize { + self.0.get() + } +} diff --git a/crates/plasm-core/src/row_plan/engine.rs b/crates/plasm-core/src/row_plan/engine.rs new file mode 100644 index 00000000..c1098e3b --- /dev/null +++ b/crates/plasm-core/src/row_plan/engine.rs @@ -0,0 +1,80 @@ +//! Engine ports. Implementations live in `plasm-runtime`. No `polars` types here. + +use crate::plasm_monad::StepId; +use crate::value::Value; +use indexmap::IndexMap; + +use super::collect::CollectReason; +use super::error::RowComputeError; +use super::ids::{EnginePlanId, FixtureScanId, FrameId, GraphSnapshotId}; +use super::plan::RowPlan; +use super::schema::PlasmFrameSchema; +use crate::identity::EntityName; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ScanSource { + Fixture { + id: FixtureScanId, + schema: PlasmFrameSchema, + }, + Inline { + schema: PlasmFrameSchema, + }, + Graph { + entity: EntityName, + snapshot: GraphSnapshotId, + schema: PlasmFrameSchema, + }, +} + +pub struct IngestBatch<'a> { + pub rows: &'a [IndexMap], +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CollectedFrame { + pub schema: PlasmFrameSchema, + pub rows: Vec>, +} + +pub trait IngestRows { + fn ingest( + &mut self, + source: &ScanSource, + batch: IngestBatch<'_>, + ) -> Result; +} + +pub trait CompileRowPlan { + fn compile(&self, plan: &RowPlan) -> Result; +} + +pub trait CollectRows { + fn collect( + &self, + id: EnginePlanId, + reason: CollectReason, + ) -> Result; +} + +/// Convenience bound for the single phase-1 adapter (not object-safe). +pub trait RowComputeEngine: IngestRows + CompileRowPlan + CollectRows {} + +impl RowComputeEngine for T where T: IngestRows + CompileRowPlan + CollectRows {} + +impl CollectedFrame { + #[must_use] + pub fn empty(schema: PlasmFrameSchema) -> Self { + Self { + schema, + rows: Vec::new(), + } + } +} + +impl CollectReason { + #[must_use] + pub fn program_return(step: StepId) -> Self { + Self::ProgramReturn { step } + } +} diff --git a/crates/plasm-core/src/row_plan/error.rs b/crates/plasm-core/src/row_plan/error.rs new file mode 100644 index 00000000..eb55db29 --- /dev/null +++ b/crates/plasm-core/src/row_plan/error.rs @@ -0,0 +1,125 @@ +//! Typed row-compute errors — no stringly engine failures. + +use crate::identity::EntityName; +use crate::money::{CrossCurrencyError, MoneyError}; +use crate::plasm_monad::ArithOp; +use thiserror::Error; + +use super::schema::LogicalColumnType; + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum RowComputeError { + #[error(transparent)] + Type(#[from] RowTypeError), + #[error(transparent)] + Money(#[from] MoneyError), + #[error("cannot compare money in {left} to money in {right}")] + CrossCurrency { left: String, right: String }, + #[error(transparent)] + Schema(#[from] FrameSchemaError), + #[error(transparent)] + Collect(#[from] CollectError), + #[error(transparent)] + Expr(#[from] crate::plasm_monad::WithExprError), + #[error(transparent)] + Predicate(#[from] RowFilterError), + #[error(transparent)] + Scan(#[from] ScanError), + #[error(transparent)] + Fusion(#[from] FusionError), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum RowTypeError { + #[error("arithmetic `{op:?}` is not defined for {lhs:?} and {rhs:?}")] + ArithDomain { + op: ArithOp, + lhs: LogicalColumnType, + rhs: LogicalColumnType, + }, + #[error("when() branches have mismatched types {then:?} vs {else_:?}")] + WhenBranchMismatch { + then: LogicalColumnType, + else_: LogicalColumnType, + }, + #[error("temporal arithmetic requires a temporal value, got {got:?}")] + TemporalArithNotTemporal { got: LogicalColumnType }, + #[error("money must not be stored as Utf8")] + MoneyStoredAsUtf8, + #[error("project spec cannot be used as a .with column")] + ProjectIntoWith, + #[error(".with must preserve entity identity")] + WithBreaksEntityShape, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum FrameSchemaError { + #[error("unknown column `{0}`")] + UnknownColumn(String), + #[error("empty pipeline is illegal")] + EmptyPipeline, + #[error("limit count must be non-zero")] + ZeroLimit, + #[error("group_by requires at least one key")] + EmptyGroupKeys, + #[error("with requires at least one column")] + EmptyWith, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CollectError { + #[error("collect is only legal at a program-return, page, invoke-arg, or render barrier")] + CollectNotAtBarrier, + #[error("render row cap exceeded: got {got}, max {max}")] + RenderRowCap { got: usize, max: usize }, + #[error("silent page exhaust is forbidden")] + PageExhaustSilent, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum RowFilterError { + #[error("row filter requires at least one predicate")] + Empty, + #[error("row filter cannot be rewritten as a catalog filter")] + CrossPlanePushdown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum ScanError { + #[error("unbound frame")] + UnboundFrame, + #[error("fixture scan `{0}` is not loaded")] + MissingFixture(u64), + #[error("entity `{0}` is not in the graph snapshot")] + MissingGraphEntity(EntityName), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum FusionError { + #[error("sort and limit must not be commuted")] + CommuteSortLimit, + #[error("optimizer must not rewrite row filters into catalog filters")] + CrossPlanePushdown, + #[error("join cannot be constructed from the surface")] + JoinFromSurface, + #[error("render is a collect barrier, not a pipeline node")] + RenderInPipeline, + #[error("derive remap cannot fold into a row-compute pipeline")] + DeriveInPipeline, +} + +impl From for RowComputeError { + fn from(e: CrossCurrencyError) -> Self { + Self::CrossCurrency { + left: e.left().to_string(), + right: e.right().to_string(), + } + } +} + +impl RowComputeError { + #[must_use] + pub fn temporal_arith_not_temporal(got: LogicalColumnType) -> Self { + Self::Type(RowTypeError::TemporalArithNotTemporal { got }) + } +} diff --git a/crates/plasm-core/src/row_plan/expr.rs b/crates/plasm-core/src/row_plan/expr.rs new file mode 100644 index 00000000..734eb54e --- /dev/null +++ b/crates/plasm-core/src/row_plan/expr.rs @@ -0,0 +1,11 @@ +//! Projection spec — distinct from `.with` columns. + +use crate::plasm_monad::payload::{FieldPath, OutputName}; +use serde::{Deserialize, Serialize}; + +pub use crate::plasm_monad::{ArithOp, WithColumn, WithExpr, WithExprError, WithLiteral}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProjectSpec { + pub fields: std::collections::BTreeMap, +} diff --git a/crates/plasm-core/src/row_plan/filter.rs b/crates/plasm-core/src/row_plan/filter.rs new file mode 100644 index 00000000..5ed542f9 --- /dev/null +++ b/crates/plasm-core/src/row_plan/filter.rs @@ -0,0 +1,50 @@ +//! Catalog vs row filter newtypes — cannot be substituted. + +use crate::plasm_monad::payload::PlanPredicate; +use serde::{Deserialize, Serialize}; + +use super::error::RowFilterError; + +/// Fetch-plane predicates (`e1{…}`). Not a row-compute input. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CatalogFilter(Vec); + +impl CatalogFilter { + #[must_use] + pub fn new(predicates: Vec) -> Self { + Self(predicates) + } + + #[must_use] + pub fn predicates(&self) -> &[PlanPredicate] { + &self.0 + } +} + +/// Row-plane AND-filter. No conversion to [`CatalogFilter`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RowFilter { + predicates: Vec, +} + +impl RowFilter { + pub fn new(predicates: Vec) -> Result { + if predicates.is_empty() { + return Err(RowFilterError::Empty); + } + Ok(Self { predicates }) + } + + #[must_use] + pub fn predicates(&self) -> &[PlanPredicate] { + &self.predicates + } +} + +impl TryFrom> for RowFilter { + type Error = RowFilterError; + + fn try_from(predicates: Vec) -> Result { + Self::new(predicates) + } +} diff --git a/crates/plasm-core/src/row_plan/fold.rs b/crates/plasm-core/src/row_plan/fold.rs new file mode 100644 index 00000000..9126d809 --- /dev/null +++ b/crates/plasm-core/src/row_plan/fold.rs @@ -0,0 +1,109 @@ +//! Fold hashed `ComputeOp` constructors into a fused [`RowPlan`]. + +use crate::plasm_monad::{ComputeOp, StepId}; + +use super::collect::{CollectCardinality, CollectReason, RenderCollectSpec}; +use super::error::{FrameSchemaError, FusionError, RowComputeError}; +use super::expr::ProjectSpec; +use super::filter::RowFilter; +use super::ids::{FrameId, RowNodeId, SurfaceMeaningId}; +use super::plan::{Pipeline, PlanNode, RowPlan, TypedAggregate}; +use std::num::NonZeroUsize; + +/// Fold a linear Map-spine `ComputeOp` chain. `Render` is a collect barrier, not a node. +pub fn fold_compute_ops( + ops: &[ComputeOp], + source: FrameId, + step: StepId, + cardinality: CollectCardinality, +) -> Result { + let meaning = SurfaceMeaningId::from_bytes( + &serde_json::to_vec(ops).unwrap_or_else(|_| ops.len().to_le_bytes().to_vec()), + ); + let mut pipeline = Pipeline::new(); + let mut collect = CollectReason::ProgramReturn { step: step.clone() }; + for (i, op) in ops.iter().enumerate() { + let id = RowNodeId::new(i as u64 + 1); + match op { + ComputeOp::Render { + columns, + template, + column_aliases, + render_bindings, + } => { + if i + 1 != ops.len() { + return Err(FusionError::RenderInPipeline.into()); + } + collect = CollectReason::Render { + step, + spec: RenderCollectSpec { + columns: columns.clone(), + column_aliases: column_aliases.clone(), + template: template.clone(), + collection_alias: None, + render_bindings: render_bindings.clone(), + }, + }; + break; + } + other => pipeline.push(id, plan_node_from_compute(other)?)?, + } + } + Ok(RowPlan::new( + source, + pipeline, + collect, + cardinality, + meaning, + )?) +} + +pub fn plan_node_from_compute(op: &ComputeOp) -> Result { + match op { + ComputeOp::Filter { predicates } => { + let filter = RowFilter::new(predicates.clone())?; + Ok(PlanNode::Filter(filter)) + } + ComputeOp::Sort { key, descending } => Ok(PlanNode::Sort { + key: key.clone(), + descending: *descending, + }), + ComputeOp::Limit { count } => { + let count = NonZeroUsize::new(*count).ok_or(FrameSchemaError::ZeroLimit)?; + Ok(PlanNode::Limit { count }) + } + ComputeOp::DedupeBy { keys } => Ok(PlanNode::Dedupe { keys: keys.clone() }), + ComputeOp::Project { fields } => Ok(PlanNode::Project(ProjectSpec { + fields: fields.clone(), + })), + ComputeOp::With { columns } => { + if columns.is_empty() { + return Err(FrameSchemaError::EmptyWith.into()); + } + Ok(PlanNode::With { + columns: columns.clone(), + }) + } + ComputeOp::GroupBy { keys, aggregates } => { + if keys.is_empty() { + return Err(FrameSchemaError::EmptyGroupKeys.into()); + } + let aggs = aggregates + .iter() + .map(TypedAggregate::from_spec) + .collect::, _>>()?; + Ok(PlanNode::GroupBy { + keys: keys.clone(), + aggs, + }) + } + ComputeOp::Aggregate { aggregates } => { + let aggs = aggregates + .iter() + .map(TypedAggregate::from_spec) + .collect::, _>>()?; + Ok(PlanNode::Aggregate { aggs }) + } + ComputeOp::Render { .. } => Err(FusionError::RenderInPipeline.into()), + } +} diff --git a/crates/plasm-core/src/row_plan/ids.rs b/crates/plasm-core/src/row_plan/ids.rs new file mode 100644 index 00000000..256e2001 --- /dev/null +++ b/crates/plasm-core/src/row_plan/ids.rs @@ -0,0 +1,67 @@ +//! Opaque identifiers for frames, fused nodes, and engine handles. +//! +//! None of these appear on hashed [`crate::PlasmComp`]. + +use serde::{Deserialize, Serialize}; + +macro_rules! u64_id { + ($(#[$meta:meta])* $name:ident) => { + $(#[$meta])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[serde(transparent)] + pub struct $name(u64); + + impl $name { + #[must_use] + pub const fn new(raw: u64) -> Self { + Self(raw) + } + + #[must_use] + pub const fn as_u64(self) -> u64 { + self.0 + } + } + }; +} + +u64_id! { + /// Session-local ingested frame. + FrameId +} + +u64_id! { + /// Node inside a fused [`super::RowPlan`] pipeline. + RowNodeId +} + +u64_id! { + /// Adapter-private compiled plan handle. Never stored on `PlasmComp`. + EnginePlanId +} + +u64_id! { + /// Language-matrix / unit-test scan. + FixtureScanId +} + +u64_id! { + /// Graph-backed scan (hot snapshot identity). + GraphSnapshotId +} + +u64_id! { + /// Hash of the surface `ComputeOp` chain (written order), not the fused engine plan. + SurfaceMeaningId +} + +impl SurfaceMeaningId { + #[must_use] + pub fn from_bytes(bytes: &[u8]) -> Self { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(bytes); + let mut raw = [0u8; 8]; + raw.copy_from_slice(&digest[..8]); + Self(u64::from_be_bytes(raw)) + } +} diff --git a/crates/plasm-core/src/row_plan/mod.rs b/crates/plasm-core/src/row_plan/mod.rs new file mode 100644 index 00000000..8d3b68ac --- /dev/null +++ b/crates/plasm-core/src/row_plan/mod.rs @@ -0,0 +1,207 @@ +//! Fused row-compute IR and engine ports. +//! +//! [`ComputeOp`](crate::ComputeOp) remains the hashed PlasmComp constructor. This module is the +//! execute-time IR. Polars types do not appear here. + +mod collect; +mod engine; +mod error; +mod expr; +mod filter; +mod fold; +mod ids; +mod plan; +mod schema; +mod with_parse; + +pub use collect::{CollectCardinality, CollectReason, PageCursor, PageSize, RenderCollectSpec}; +pub use engine::{ + CollectRows, CollectedFrame, CompileRowPlan, IngestBatch, IngestRows, RowComputeEngine, + ScanSource, +}; +pub use error::{ + CollectError, FrameSchemaError, FusionError, RowComputeError, RowFilterError, RowTypeError, + ScanError, +}; +pub use expr::{ArithOp, ProjectSpec, WithColumn, WithExpr, WithExprError, WithLiteral}; +pub use filter::{CatalogFilter, RowFilter}; +pub use fold::{fold_compute_ops, plan_node_from_compute}; +pub use ids::{EnginePlanId, FixtureScanId, FrameId, GraphSnapshotId, RowNodeId, SurfaceMeaningId}; +pub use plan::{MoneyAggLaw, NumericAgg, Pipeline, PlanNode, RowPlan, TypedAggregate}; +pub use schema::{ + ColumnName, FrameShape, IdentityPreservation, LogicalColumn, LogicalColumnType, + MoneyColumnLayout, PlasmFrameSchema, RemapReason, +}; +pub use with_parse::parse_with_body; + +#[cfg(test)] +mod tests { + use super::*; + use crate::plasm_monad::payload::PlasmDataValue; + use crate::plasm_monad::{ComputeOp, FieldPath, OutputName, PlanPredicate, PlanPredicateOp}; + + #[test] + fn plan_node_has_no_render_or_join_variants() { + let names: Vec<&str> = vec![ + "Filter", + "Sort", + "Limit", + "Dedupe", + "Distinct", + "Project", + "With", + "GroupBy", + "Aggregate", + ]; + assert!(!names.contains(&"Render")); + assert!(!names.contains(&"EquiJoin")); + assert!(!names.contains(&"Join")); + } + + #[test] + fn with_parse_now_minus_and_mul() { + let cols = parse_with_body("age_days: (now - updated_at), notional: quantity * price") + .expect("parse"); + assert_eq!(cols.len(), 2); + match &cols[0].expr { + WithExpr::Arith { + op: ArithOp::Sub, + lhs, + rhs, + } => { + assert!(matches!(lhs.as_ref(), WithExpr::Now)); + assert!(matches!(rhs.as_ref(), WithExpr::Field(_))); + } + other => panic!("expected now - field, got {other:?}"), + } + assert!(matches!( + cols[1].expr, + WithExpr::Arith { + op: ArithOp::Mul, + .. + } + )); + } + + #[test] + fn with_parse_div_concat_when_and_field_minus_field() { + let cols = parse_with_body( + "cycle: (updated_at - created_at), rate: qty / n, name: first + last, blank: when(len(title)=0, 1, 0), stale: when(now - updated_at > 14, 1, 0)", + ) + .expect("parse"); + assert_eq!(cols.len(), 5); + assert!(matches!( + &cols[0].expr, + WithExpr::Arith { + op: ArithOp::Sub, + lhs, + rhs, + } if matches!(lhs.as_ref(), WithExpr::Field(_)) && matches!(rhs.as_ref(), WithExpr::Field(_)) + )); + assert!(matches!( + cols[1].expr, + WithExpr::Arith { + op: ArithOp::Div, + .. + } + )); + assert!(matches!( + cols[2].expr, + WithExpr::Arith { + op: ArithOp::Add, + .. + } + )); + match &cols[3].expr { + WithExpr::When { lhs, op, rhs, .. } => { + assert!(matches!(lhs.as_ref(), WithExpr::Len { .. })); + assert_eq!(*op, PlanPredicateOp::Eq); + assert!(matches!( + rhs.as_ref(), + WithExpr::Literal(WithLiteral::Integer(0)) + )); + } + other => panic!("expected when(len), got {other:?}"), + } + match &cols[4].expr { + WithExpr::When { lhs, op, .. } => { + assert!(matches!( + lhs.as_ref(), + WithExpr::Arith { + op: ArithOp::Sub, + .. + } + )); + assert_eq!(*op, PlanPredicateOp::Gt); + } + other => panic!("expected when(now - field), got {other:?}"), + } + } + + #[test] + fn fold_render_is_collect_barrier() { + let op = ComputeOp::Render { + columns: vec![OutputName::new("title").unwrap()], + template: "{{ r.title }}".into(), + column_aliases: Default::default(), + render_bindings: vec![], + }; + let plan = fold_compute_ops( + &[op], + FrameId::new(1), + crate::plasm_monad::StepId::new("out").unwrap(), + CollectCardinality::List, + ) + .unwrap(); + assert!(matches!(plan.collect(), CollectReason::Render { .. })); + assert!(plan.nodes().is_empty()); + } + + #[test] + fn fold_filter_does_not_become_catalog_filter() { + let pred = PlanPredicate { + field_path: FieldPath::from_dotted("owner").unwrap(), + op: PlanPredicateOp::Eq, + value: PlasmDataValue::Literal { + value: serde_json::json!("alice"), + }, + }; + let node = plan_node_from_compute(&ComputeOp::Filter { + predicates: vec![pred], + }) + .unwrap(); + assert!(matches!(node, PlanNode::Filter(_))); + } + + #[test] + fn with_body_rejects_empty() { + assert!(parse_with_body("").is_err()); + assert!(parse_with_body(" ").is_err()); + } + + #[test] + fn with_body_rejects_hop_summary_and_unknown_calls() { + let err = parse_with_body("n: count(r1)").unwrap_err().to_string(); + assert!(err.contains("count"), "{err}"); + assert!(parse_with_body("x: open(labels)").is_err()); + assert!(parse_with_body("x: rank(score)").is_err()); + let age = parse_with_body("x: age_days(updated_at)") + .unwrap_err() + .to_string(); + assert!(age.contains("age_days"), "{age}"); + assert!(age.contains("len"), "{age}"); + assert!(age.contains("when"), "{age}"); + assert!(!age.contains("age_days, len"), "{age}"); + let empty = parse_with_body("x: empty(title)").unwrap_err().to_string(); + assert!(empty.contains("empty"), "{empty}"); + assert!(parse_with_body("x: datediff(updated_at)").is_err()); + assert!(parse_with_body("x: col(updated_at)").is_err()); + } + + #[test] + fn fusion_error_join_from_surface_is_named() { + let msg = FusionError::JoinFromSurface.to_string(); + assert!(msg.contains("join")); + assert!(msg.contains("surface")); + } +} diff --git a/crates/plasm-core/src/row_plan/plan.rs b/crates/plasm-core/src/row_plan/plan.rs new file mode 100644 index 00000000..5291be95 --- /dev/null +++ b/crates/plasm-core/src/row_plan/plan.rs @@ -0,0 +1,215 @@ +//! Fused row-compute IR. EquiJoin and Render are not pipeline nodes. + +use crate::plasm_monad::payload::{AggregateSpec, FieldPath}; +use crate::plasm_monad::OutputName; +use serde::{Deserialize, Serialize}; +use std::num::NonZeroUsize; + +use super::collect::{CollectCardinality, CollectReason}; +use super::error::{FrameSchemaError, FusionError}; +use super::expr::{ProjectSpec, WithColumn}; +use super::filter::RowFilter; +use super::ids::{FrameId, RowNodeId, SurfaceMeaningId}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RowPlan { + source: FrameId, + nodes: Pipeline, + collect: CollectReason, + cardinality: CollectCardinality, + meaning: SurfaceMeaningId, +} + +impl RowPlan { + pub fn new( + source: FrameId, + nodes: Pipeline, + collect: CollectReason, + cardinality: CollectCardinality, + meaning: SurfaceMeaningId, + ) -> Result { + if nodes.is_empty() && !matches!(collect, CollectReason::Render { .. }) { + // Identity collect (return ingested rows) is legal. + } + Ok(Self { + source, + nodes, + collect, + cardinality, + meaning, + }) + } + + #[must_use] + pub fn source(&self) -> FrameId { + self.source + } + + #[must_use] + pub fn nodes(&self) -> &Pipeline { + &self.nodes + } + + #[must_use] + pub fn collect(&self) -> &CollectReason { + &self.collect + } + + #[must_use] + pub fn cardinality(&self) -> CollectCardinality { + self.cardinality + } + + #[must_use] + pub fn meaning(&self) -> SurfaceMeaningId { + self.meaning + } +} + +/// Append-only pipeline. Written order is meaning; no swap/insert. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct Pipeline(Vec<(RowNodeId, PlanNode)>); + +impl Pipeline { + #[must_use] + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn push(&mut self, id: RowNodeId, node: PlanNode) -> Result<(), FusionError> { + self.0.push((id, node)); + Ok(()) + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[must_use] + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlanNode { + Filter(RowFilter), + Sort { + key: FieldPath, + descending: bool, + }, + Limit { + count: NonZeroUsize, + }, + Dedupe { + keys: Vec, + }, + Distinct { + keys: Vec, + }, + Project(ProjectSpec), + With { + columns: Vec, + }, + GroupBy { + keys: Vec, + aggs: Vec, + }, + Aggregate { + aggs: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TypedAggregate { + Count { + name: OutputName, + }, + Numeric { + name: OutputName, + fn_: NumericAgg, + field: FieldPath, + }, + MoneySum { + name: OutputName, + field: FieldPath, + currency: MoneyAggLaw, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NumericAgg { + Sum, + Avg, + Min, + Max, + First, + Last, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MoneyAggLaw { + RequireUniform, + CurrencyIsGroupKey, +} + +impl TypedAggregate { + pub fn from_spec(spec: &AggregateSpec) -> Result { + use crate::plasm_monad::AggregateFunction; + match spec.function { + AggregateFunction::Count => Ok(Self::Count { + name: spec.name.clone(), + }), + AggregateFunction::Sum => { + let field = spec + .field + .clone() + .ok_or(FrameSchemaError::UnknownColumn("sum field".into()))?; + Ok(Self::Numeric { + name: spec.name.clone(), + fn_: NumericAgg::Sum, + field, + }) + } + AggregateFunction::Avg => map_numeric(spec, NumericAgg::Avg), + AggregateFunction::Min => map_numeric(spec, NumericAgg::Min), + AggregateFunction::Max => map_numeric(spec, NumericAgg::Max), + AggregateFunction::First => map_numeric(spec, NumericAgg::First), + AggregateFunction::Last => map_numeric(spec, NumericAgg::Last), + } + } + + #[must_use] + pub fn as_money_sum(spec: &AggregateSpec) -> Option { + use crate::plasm_monad::AggregateFunction; + if spec.function != AggregateFunction::Sum { + return None; + } + spec.field.clone().map(|field| Self::MoneySum { + name: spec.name.clone(), + field, + currency: MoneyAggLaw::RequireUniform, + }) + } +} + +fn map_numeric(spec: &AggregateSpec, fn_: NumericAgg) -> Result { + let field = spec + .field + .clone() + .ok_or(FrameSchemaError::UnknownColumn(format!("{fn_:?} field")))?; + Ok(TypedAggregate::Numeric { + name: spec.name.clone(), + fn_, + field, + }) +} diff --git a/crates/plasm-core/src/row_plan/schema.rs b/crates/plasm-core/src/row_plan/schema.rs new file mode 100644 index 00000000..06dac95b --- /dev/null +++ b/crates/plasm-core/src/row_plan/schema.rs @@ -0,0 +1,141 @@ +//! Logical frame schema. Physical Polars dtypes stay behind the runtime adapter. + +use crate::identity::EntityName; +use crate::plasm_monad::payload::FieldPath; +use crate::TemporalWireFormat; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; + +use crate::plasm_monad::OutputName; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ColumnName(OutputName); + +impl ColumnName { + pub fn new(name: impl Into) -> Result { + Ok(Self(OutputName::new(name.into())?)) + } + + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + #[must_use] + pub fn as_output_name(&self) -> &OutputName { + &self.0 + } +} + +impl From for ColumnName { + fn from(name: OutputName) -> Self { + Self(name) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlasmFrameSchema { + shape: FrameShape, + columns: IndexMap, +} + +impl PlasmFrameSchema { + #[must_use] + pub fn new(shape: FrameShape, columns: IndexMap) -> Self { + Self { shape, columns } + } + + #[must_use] + pub fn opaque_object() -> Self { + Self { + shape: FrameShape::Remapped { + reason: RemapReason::Project, + }, + columns: IndexMap::new(), + } + } + + #[must_use] + pub fn shape(&self) -> &FrameShape { + &self.shape + } + + #[must_use] + pub fn columns(&self) -> &IndexMap { + &self.columns + } + + #[must_use] + pub fn with_intact_identity(mut self) -> Self { + if let FrameShape::Entity { identity, .. } = &mut self.shape { + *identity = IdentityPreservation::Intact; + } + self + } + + pub fn insert_column(&mut self, name: ColumnName, col: LogicalColumn) { + self.columns.insert(name.as_str().to_string(), col); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FrameShape { + Entity { + entity: EntityName, + identity: IdentityPreservation, + }, + Remapped { + reason: RemapReason, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentityPreservation { + Intact, + Projected, + Aggregated, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RemapReason { + Project, + GroupBy, + Aggregate, + Derive, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LogicalColumn { + pub ty: LogicalColumnType, + pub nullable: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogicalColumnType { + Null, + Boolean, + Integer, + Number, + String, + Duration, + Temporal { format: TemporalWireFormat }, + Money { currency: MoneyColumnLayout }, + EntityRef { target: EntityName }, + Array, + Object, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MoneyColumnLayout { + Uniform { currency: String }, + PerRow, +} diff --git a/crates/plasm-core/src/row_plan/with_parse.rs b/crates/plasm-core/src/row_plan/with_parse.rs new file mode 100644 index 00000000..f68937a1 --- /dev/null +++ b/crates/plasm-core/src/row_plan/with_parse.rs @@ -0,0 +1,232 @@ +//! Parse `.with{name: expr, …}` bodies. + +use crate::plasm_monad::payload::{FieldPath, PlanPredicateOp}; +use crate::plasm_monad::{OutputName, WithColumn, WithExpr, WithExprError, WithLiteral}; + +use super::expr::ArithOp; + +pub fn parse_with_body(body: &str) -> Result, WithExprError> { + let body = body.trim(); + if body.is_empty() { + return Err(WithExprError::EmptyBody); + } + let mut columns = Vec::new(); + for part in split_top_level_comma(body) { + let part = part.trim(); + let Some((name, expr)) = part.split_once(':') else { + return Err(WithExprError::Parse(format!( + "expected `name: expr`, got `{part}`" + ))); + }; + let name = OutputName::new(name.trim().to_string()).map_err(WithExprError::BadColumn)?; + let expr = parse_with_expr(expr.trim())?; + columns.push(WithColumn { name, expr }); + } + if columns.is_empty() { + return Err(WithExprError::EmptyBody); + } + Ok(columns) +} + +fn split_top_level_comma(s: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut depth = 0i32; + let mut start = 0usize; + for (i, c) in s.char_indices() { + match c { + '(' | '{' => depth += 1, + ')' | '}' => depth -= 1, + ',' if depth == 0 => { + out.push(&s[start..i]); + start = i + 1; + } + _ => {} + } + } + out.push(&s[start..]); + out +} + +fn parse_with_expr(s: &str) -> Result { + let s = s.trim(); + parse_arith(s) +} + +fn parse_arith(s: &str) -> Result { + if let Some((lhs, rhs)) = split_top_bin(s, '+') { + return Ok(WithExpr::Arith { + op: ArithOp::Add, + lhs: Box::new(parse_arith(lhs)?), + rhs: Box::new(parse_arith(rhs)?), + }); + } + if let Some((lhs, rhs)) = split_top_bin(s, '-') { + if !lhs.trim().is_empty() { + return Ok(WithExpr::Arith { + op: ArithOp::Sub, + lhs: Box::new(parse_arith(lhs)?), + rhs: Box::new(parse_arith(rhs)?), + }); + } + } + if let Some((op, lhs, rhs)) = split_top_muldiv(s) { + return Ok(WithExpr::Arith { + op, + lhs: Box::new(parse_arith(lhs)?), + rhs: Box::new(parse_arith(rhs)?), + }); + } + parse_atom(s) +} + +fn split_top_bin(s: &str, op: char) -> Option<(&str, &str)> { + let mut depth = 0i32; + for (i, c) in s.char_indices().rev() { + match c { + ')' | '}' => depth += 1, + '(' | '{' => depth -= 1, + c if c == op && depth == 0 && i > 0 => { + return Some((s[..i].trim(), s[i + op.len_utf8()..].trim())); + } + _ => {} + } + } + None +} + +fn split_top_muldiv(s: &str) -> Option<(ArithOp, &str, &str)> { + let mut depth = 0i32; + for (i, c) in s.char_indices().rev() { + match c { + ')' | '}' => depth += 1, + '(' | '{' => depth -= 1, + '*' | '/' if depth == 0 && i > 0 => { + let op = if c == '*' { ArithOp::Mul } else { ArithOp::Div }; + return Some((op, s[..i].trim(), s[i + 1..].trim())); + } + _ => {} + } + } + None +} + +fn parse_atom(s: &str) -> Result { + let s = s.trim(); + if let Some(inner) = strip_wrapping_parens(s) { + return parse_with_expr(inner); + } + if s.eq_ignore_ascii_case("null") { + return Ok(WithExpr::Literal(WithLiteral::Null)); + } + if s.eq_ignore_ascii_case("true") { + return Ok(WithExpr::Literal(WithLiteral::Bool(true))); + } + if s.eq_ignore_ascii_case("false") { + return Ok(WithExpr::Literal(WithLiteral::Bool(false))); + } + if s.eq_ignore_ascii_case("now") { + return Ok(WithExpr::Now); + } + if let Some(inner) = s.strip_prefix('"').and_then(|t| t.strip_suffix('"')) { + return Ok(WithExpr::Literal(WithLiteral::String(inner.to_string()))); + } + if let Some(rest) = s.strip_prefix("len(").and_then(|t| t.strip_suffix(')')) { + return Ok(WithExpr::Len { + field: FieldPath::from_dotted(rest.trim()).map_err(WithExprError::Parse)?, + }); + } + if let Some(rest) = s.strip_prefix("when(").and_then(|t| t.strip_suffix(')')) { + return parse_when(rest); + } + if let Ok(i) = s.parse::() { + return Ok(WithExpr::Literal(WithLiteral::Integer(i))); + } + if s.parse::().is_ok() { + return Ok(WithExpr::Literal(WithLiteral::Number(s.to_string()))); + } + if let Some(idx) = s.find('(') { + if s.ends_with(')') { + let fname = &s[..idx]; + return Err(WithExprError::Parse(format!( + "unknown .with function `{fname}` (known calls: len, when; `now` is a word, not a call)" + ))); + } + } + FieldPath::from_dotted(s) + .map(WithExpr::Field) + .map_err(WithExprError::Parse) +} + +/// Outer `(`…`)` only when that pair wraps the whole atom (`(now - t)`, not `(a)+(b)`). +fn strip_wrapping_parens(s: &str) -> Option<&str> { + if !s.starts_with('(') || !s.ends_with(')') { + return None; + } + let mut depth = 0i32; + for (i, c) in s.char_indices() { + match c { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + if i + 1 == s.len() { + return Some(s[1..i].trim()); + } + return None; + } + } + _ => {} + } + } + None +} + +fn parse_when(args: &str) -> Result { + let parts = split_top_level_comma(args); + if parts.len() != 3 { + return Err(WithExprError::Parse( + "when(pred, then, else) requires three arguments".into(), + )); + } + let (lhs, op, rhs) = split_when_cmp(parts[0].trim())?; + Ok(WithExpr::When { + lhs: Box::new(parse_with_expr(lhs)?), + op, + rhs: Box::new(parse_with_expr(rhs)?), + then: Box::new(parse_with_expr(parts[1])?), + else_: Box::new(parse_with_expr(parts[2])?), + }) +} + +fn split_when_cmp(s: &str) -> Result<(&str, PlanPredicateOp, &str), WithExprError> { + let ops: [(&str, PlanPredicateOp); 6] = [ + (">=", PlanPredicateOp::Gte), + ("<=", PlanPredicateOp::Lte), + ("!=", PlanPredicateOp::Ne), + ("=", PlanPredicateOp::Eq), + (">", PlanPredicateOp::Gt), + ("<", PlanPredicateOp::Lt), + ]; + let mut depth = 0i32; + for (i, c) in s.char_indices() { + match c { + '(' | '{' => depth += 1, + ')' | '}' => depth -= 1, + _ if depth == 0 && i > 0 => { + for (sym, op) in ops { + if s[i..].starts_with(sym) { + let lhs = s[..i].trim(); + let rhs = s[i + sym.len()..].trim(); + if !lhs.is_empty() && !rhs.is_empty() { + return Ok((lhs, op, rhs)); + } + } + } + } + _ => {} + } + } + Err(WithExprError::Parse(format!( + "when() predicate must be a comparison, got `{s}`" + ))) +} diff --git a/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap b/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap index 31f3b8f8..7e2b2153 100644 --- a/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap +++ b/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap @@ -1,5 +1,6 @@ --- -source: crates/plasm-core/src/prompt_render/tests.rs +source: plasm-oss/crates/plasm-core/src/prompt_render/tests.rs +assertion_line: 1972 expression: "super::PLASM_TOOL_DESCRIPTION" --- **Plasm** — **`logical_session_ref`** + **`program`**. Clean read-only plans execute inline (rows in tool **`content`**). Plans with any mutation / review gate return a **`run_ref`** (`pcN`, 10 min TTL) for **`plasm_run`**; do **not** echo the program. @@ -45,7 +46,7 @@ TSV table semantics: Core surface: - Get identity: `e#(id)` (parens). Query/filter: `e#{field=…}` (braces). Search when taught: `e#~$` / `e#~"text"`. -- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `[field,…]`. +- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `.with{k: expr}` `[field,…]`. - Inline rows when delivery is `inline` (≤25); `snapshot_only` / `(in artifact)` need artifact read. Copy **`artifact_uri`** from the step — never plan `run_step` / `dict_ref`. - `page(...)` is HTTP-execute only — not an MCP tool argument. diff --git a/crates/plasm-core/src/typed_row.rs b/crates/plasm-core/src/typed_row.rs index 994bd14f..1f7bbb28 100644 --- a/crates/plasm-core/src/typed_row.rs +++ b/crates/plasm-core/src/typed_row.rs @@ -22,6 +22,8 @@ pub enum TypedFieldValue { Object(IndexMap), /// Normalized `entity_ref` payload when [`FieldType::EntityRef`] applies and the wire shape parses. EntityRef(EntityRefPayload), + /// Fowler money — must not collapse to [`TypedFieldValue::Json`]. + Money(crate::money::MoneyValue), PlasmInputRef(PlasmInputRef), /// Arbitrary subtree (`FieldType::Json`, `Blob`, attachment blobs) stored verbatim. Json(Value), @@ -39,6 +41,10 @@ impl TypedFieldValue { pub fn from_value_in_field(field_type: &FieldType, v: Value) -> Self { match field_type { FieldType::Json | FieldType::Blob => Self::Json(v), + FieldType::Money => match v { + Value::Money(m) => Self::Money(m), + other => Self::from(other), + }, FieldType::EntityRef { .. } => match EntityRefPayload::try_from_value(&v) { Ok(p) => Self::EntityRef(p), Err(_) => Self::from(v), @@ -61,6 +67,7 @@ impl TypedFieldValue { Value::Object(m.iter().map(|(k, v)| (k.clone(), v.to_value())).collect()) } TypedFieldValue::EntityRef(p) => p.to_value(), + TypedFieldValue::Money(m) => Value::Money(m.clone()), TypedFieldValue::PlasmInputRef(r) => Value::PlasmInputRef(r.clone()), TypedFieldValue::Json(v) => v.clone(), } @@ -107,7 +114,7 @@ impl From for TypedFieldValue { Value::Float(f) => TypedFieldValue::Float(f), Value::String(s) | Value::PhraseIdent(s) => TypedFieldValue::String(s), Value::Array(a) => TypedFieldValue::Array(a.into_iter().map(Self::from).collect()), - Value::Money(_) => TypedFieldValue::Json(v), + Value::Money(m) => TypedFieldValue::Money(m), Value::Object(m) => { TypedFieldValue::Object(m.into_iter().map(|(k, v)| (k, Self::from(v))).collect()) } @@ -190,4 +197,19 @@ mod tests { other => panic!("expected Json variant: {other:?}"), } } + + #[test] + fn money_does_not_dump_to_json() { + let m = + crate::money::MoneyValue::new(rust_decimal::Decimal::new(1250, 2), Some("USD".into())); + let v = Value::Money(m.clone()); + let tf = TypedFieldValue::from(v); + match tf { + TypedFieldValue::Money(got) => assert_eq!(got, m), + other => panic!("expected Money variant: {other:?}"), + } + let tf2 = TypedFieldValue::from_value_in_field(&FieldType::Money, Value::Money(m.clone())); + assert!(matches!(tf2, TypedFieldValue::Money(_))); + assert_eq!(tf2.to_value(), Value::Money(m)); + } } diff --git a/crates/plasm-e2e/tests/plasm_language_matrix.rs b/crates/plasm-e2e/tests/plasm_language_matrix.rs index 64301690..a72f8c29 100644 --- a/crates/plasm-e2e/tests/plasm_language_matrix.rs +++ b/crates/plasm-e2e/tests/plasm_language_matrix.rs @@ -10,7 +10,7 @@ //! //! - Entity roots: bare query, search `~`, get `(id)`, brace predicates `{field=value}`, comparisons. //! - Postfix: `.limit`, `.sort(field[, dir])` including `asc`/`desc`, `.aggregate` (named + sugar), -//! `.group_by`, `.singleton()`, `.page_size`, bracket projection `[…]`. +//! `.group_by`, `.with{k: expr}`, `.singleton()`, `.page_size`, bracket projection `[…]`. //! - Programs: bindings, node-ref continuation, parallel final roots, `compile_plasm_expression` //! (single-line surface) vs multi-line DAG programs. //! - Relations: `from_parent_get`, `query_scoped`, opaque `r#` nav (not `p#`), one-cardinality `r#`, @@ -25,9 +25,8 @@ //! OpenAPI `example` literals. **Live `run_markdown`** is fenced TSV for row-shaped HTTP results //! ([`mcp_format_execute_result_table_or_tsv`](../../plasm-agent-core/src/mcp_run_markdown.rs)); //! operation display strings (`Query(…)`, `Get(…)`) are asserted on dry-run IR in [`assert_planning_ir`]. -//! Multi-digit **numeric** `.sort` ordering is covered in -//! `plasm-agent-core` (`plan_sort_compute_orders_integer_scores_numerically`) because Hermit list -//! payloads are not example-stable. +//! Multi-digit **numeric** `.sort` ordering is covered in `plasm-runtime` row-compute tests because +//! Hermit list payloads are not example-stable. //! //! **Planning:** dry-run [`DryPlasmPlanEvaluation::node_results`] `ir.expr` JSON is deserialized into //! typed [`plasm_core::Expr`]; compute stages deserialize into [`plasm_agent::plasm_plan::ComputeOp`]. @@ -97,6 +96,7 @@ const REQUIRED_FEATURE_TAGS: &[&str] = &[ "for_each_effect", "domain_symbol_e1", "postfix_group_by", + "postfix_with", "postfix_group_by_aggregate_chain", "postfix_row_filter", "postfix_group_by_sugar", @@ -614,6 +614,14 @@ fn assert_planning_ir( return Err(format!("expected Filter compute, got {:?}", computes)); } } + "lang_with_mul" | "lang_with_div" | "lang_with_concat" | "lang_with_when_len" => { + if !computes + .iter() + .any(|c| matches!(c.op, ComputeOp::With { .. })) + { + return Err(format!("expected With compute, got {:?}", computes)); + } + } "lang_group_by" => { let Some(ComputeTemplate { op: ComputeOp::GroupBy { keys, aggregates }, @@ -1848,6 +1856,46 @@ newbranch, newfile"#, min_node_results: 1, expect_markdown_substrings: &["```tsv", "alice"], }, + MatrixRow { + id: "lang_with_mul", + program: "items = LangItem\nboosted = items.with{boost: score * 2}.limit(3)\nboosted[id,boost]", + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "boost"], + }, + MatrixRow { + id: "lang_with_div", + program: "items = LangItem\nhalved = items.with{half: score / 2}.limit(3)\nhalved[id,half]", + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "half"], + }, + MatrixRow { + id: "lang_with_concat", + program: r#"items = LangItem.filter{owner="alice"} +tagged = items.with{tag: owner + owner}.limit(1) +tagged[tag]"#, + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit", "postfix_row_filter"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "alicealice"], + }, + MatrixRow { + id: "lang_with_when_len", + program: r#"items = LangItem.filter{owner="alice"} +labeled = items.with{label: when(len(owner)>0, owner, title)}.limit(1) +labeled[label]"#, + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit", "postfix_row_filter"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "alice"], + }, MatrixRow { id: "lang_relation_lines", program: r#"LangItem("i1").lines[id,note]"#, diff --git a/crates/plasm-runtime/Cargo.toml b/crates/plasm-runtime/Cargo.toml index b0856b67..3cb90cc5 100644 --- a/crates/plasm-runtime/Cargo.toml +++ b/crates/plasm-runtime/Cargo.toml @@ -34,6 +34,9 @@ oauth2 = { workspace = true } url = { workspace = true } opentelemetry = { workspace = true } minijinja = { version = "2.19.0", default-features = false, features = ["builtins", "serde"] } +polars = { workspace = true } +chrono = { workspace = true } +rust_decimal = { workspace = true } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/plasm-runtime/src/lib.rs b/crates/plasm-runtime/src/lib.rs index 860e26f9..bb90856b 100644 --- a/crates/plasm-runtime/src/lib.rs +++ b/crates/plasm-runtime/src/lib.rs @@ -123,6 +123,7 @@ pub mod paginated_collect; pub mod preflight; pub mod query_index; pub mod replay; +pub mod row_compute; pub mod row_predicate; pub mod runtime_error_render; pub mod session_graph_cache; @@ -198,6 +199,7 @@ pub use oauth_client::{ pub use oauth_token_debug::TokenEndpointResponseSummary; pub use query_index::{QueryCacheKey, QueryIndex}; pub use replay::*; +pub use row_compute::{eval_compute_ops, ComputeEvalOutcome, PolarsAdapter}; pub use row_predicate::{ json_matches_predicate, json_predicate_matches, JsonRowPredicate, JsonRowPredicateOp, }; diff --git a/crates/plasm-runtime/src/row_compute/adapter.rs b/crates/plasm-runtime/src/row_compute/adapter.rs new file mode 100644 index 00000000..6a1120ba --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/adapter.rs @@ -0,0 +1,112 @@ +//! Three sync engine ports. Polars types do not escape this module. + +use super::eval::collect_plan_rows; +use super::json_frame::{ingest_json_rows, FrameState}; +use indexmap::IndexMap; +use plasm_core::{ + CollectReason, CollectRows, CollectedFrame, CompileRowPlan, EnginePlanId, FrameId, IngestBatch, + IngestRows, PlasmFrameSchema, RowComputeError, RowPlan, ScanError, ScanSource, Value, +}; +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; + +/// Polars-backed row engine. Handles are session-local and never stored on `PlasmComp`. +pub struct PolarsAdapter { + frames: RefCell>, + plans: RefCell>, + next_frame: Cell, + next_engine: Cell, +} + +impl Default for PolarsAdapter { + fn default() -> Self { + Self::new() + } +} + +impl PolarsAdapter { + #[must_use] + pub fn new() -> Self { + Self { + frames: RefCell::new(HashMap::new()), + plans: RefCell::new(HashMap::new()), + next_frame: Cell::new(1), + next_engine: Cell::new(1), + } + } + + fn json_from_values(rows: &[IndexMap]) -> Vec { + rows.iter() + .map(|row| { + let mut map = serde_json::Map::new(); + for (k, v) in row { + map.insert(k.clone(), plasm_core::plasm_value_to_json(v)); + } + serde_json::Value::Object(map) + }) + .collect() + } +} + +impl IngestRows for PolarsAdapter { + fn ingest( + &mut self, + _source: &ScanSource, + batch: IngestBatch<'_>, + ) -> Result { + let json_rows = Self::json_from_values(batch.rows); + let state = ingest_json_rows(&json_rows).map_err(|_| ScanError::UnboundFrame)?; + let id = FrameId::new(self.next_frame.get()); + self.next_frame.set(id.as_u64() + 1); + self.frames.borrow_mut().insert(id, state); + Ok(id) + } +} + +impl CompileRowPlan for PolarsAdapter { + fn compile(&self, plan: &RowPlan) -> Result { + let id = EnginePlanId::new(self.next_engine.get()); + self.next_engine.set(id.as_u64() + 1); + self.plans.borrow_mut().insert(id, plan.clone()); + Ok(id) + } +} + +impl CollectRows for PolarsAdapter { + fn collect( + &self, + id: EnginePlanId, + _reason: CollectReason, + ) -> Result { + let plans = self.plans.borrow(); + let plan = plans.get(&id).ok_or(ScanError::UnboundFrame)?; + let frames = self.frames.borrow(); + let state = frames + .get(&plan.source()) + .cloned() + .ok_or(ScanError::UnboundFrame)?; + drop(frames); + let rows_json = collect_plan_rows(plan, &state).map_err(|_| ScanError::UnboundFrame)?; + let rows = rows_json + .into_iter() + .map(|v| match v { + serde_json::Value::Object(map) => map + .into_iter() + .map(|(k, val)| (k, plasm_core::json_value_to_plasm_value(&val))) + .collect(), + other => { + let mut m = IndexMap::new(); + m.insert( + "value".into(), + plasm_core::json_value_to_plasm_value(&other), + ); + m + } + }) + .collect(); + Ok(CollectedFrame { + schema: PlasmFrameSchema::opaque_object(), + rows, + }) + } +} diff --git a/crates/plasm-runtime/src/row_compute/eval.rs b/crates/plasm-runtime/src/row_compute/eval.rs new file mode 100644 index 00000000..b88004ff --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/eval.rs @@ -0,0 +1,940 @@ +//! Apply a fused [`RowPlan`] on an ingested frame. + +use super::json_frame::{ + col_expr, collect_json, ingest_json_rows, ColKind, FrameState, IDX_COL, MONEY_AMOUNT, MONEY_CCY, +}; +use chrono::{DateTime, Utc}; +use plasm_core::plasm_monad::{ + ComputeOp, FieldPath, PlanPredicate, PlanPredicateOp, PlasmDataValue, WithExpr, WithLiteral, +}; +use plasm_core::{ + fold_compute_ops, normalize_temporal_value, ArithOp, CollectCardinality, CollectReason, + FrameId, PlanNode, RowPlan, StepId, TemporalWireFormat, TypedAggregate, +}; +use polars::prelude::*; +use rust_decimal::Decimal; + +/// Engine collect before host Minijinja (Render is not a PlanNode). +#[derive(Debug, Clone)] +pub enum ComputeEvalOutcome { + Rows(Vec), + Render { + rows: Vec, + columns: Vec, + column_aliases: std::collections::BTreeMap, + template: String, + collection_alias: Option, + render_bindings: Vec, + }, +} + +pub fn eval_compute_ops( + ops: &[ComputeOp], + rows: &[serde_json::Value], +) -> Result { + let (plan, collected) = collect_ops_rows(ops, rows)?; + match plan.collect() { + CollectReason::Render { spec, .. } => Ok(ComputeEvalOutcome::Render { + rows: collected, + columns: spec.columns.clone(), + column_aliases: spec.column_aliases.clone(), + template: spec.template.clone(), + collection_alias: spec.collection_alias.clone(), + render_bindings: spec.render_bindings.clone(), + }), + _ => Ok(ComputeEvalOutcome::Rows(collected)), + } +} + +fn collect_ops_rows( + ops: &[ComputeOp], + rows: &[serde_json::Value], +) -> Result<(RowPlan, Vec), String> { + let step = StepId::new("row").map_err(|e| e.to_string())?; + let plan = fold_compute_ops(ops, FrameId::new(1), step, CollectCardinality::List) + .map_err(|e| e.to_string())?; + let state = ingest_json_rows(rows).map_err(|e| e.to_string())?; + let collected = collect_plan_rows(&plan, &state).map_err(|e| e.to_string())?; + Ok((plan, collected)) +} + +pub(super) fn collect_plan_rows( + plan: &RowPlan, + initial_state: &FrameState, +) -> PolarsResult> { + let mut state = initial_state.clone(); + apply_stored_plan(plan, &mut state)?; + collect_json(&state) +} + +pub(super) fn apply_stored_plan(plan: &RowPlan, state: &mut FrameState) -> PolarsResult<()> { + let now = Utc::now(); + ensure_plan_columns(plan, state)?; + let mut lf = state.df.clone().lazy(); + for (_, node) in plan.nodes().iter() { + lf = apply_node(lf, node, state, now)?; + } + state.df = lf.collect()?; + finalize_money_sums(state)?; + Ok(()) +} + +fn ensure_plan_columns(plan: &RowPlan, state: &mut FrameState) -> PolarsResult<()> { + let mut names = Vec::new(); + for (_, node) in plan.nodes().iter() { + collect_node_columns(node, &mut names); + } + let height = state.df.height(); + for name in names { + if state.df.column(&name).is_ok() { + continue; + } + let series = Series::full_null(PlSmallStr::from_str(&name), height, &DataType::Null); + state.df.with_column(series)?; + state.ensure_visible_kind(name, ColKind::Json); + } + Ok(()) +} + +fn collect_node_columns(node: &PlanNode, names: &mut Vec) { + match node { + PlanNode::Filter(filter) => { + for p in filter.predicates() { + names.push(p.field_path.dotted()); + } + } + PlanNode::Sort { key, .. } => names.push(key.dotted()), + PlanNode::GroupBy { keys, aggs } => { + names.extend(keys.iter().map(|k| k.dotted())); + for agg in aggs { + collect_agg_columns(agg, names); + } + } + PlanNode::Aggregate { aggs } => { + for agg in aggs { + collect_agg_columns(agg, names); + } + } + PlanNode::Project(spec) => { + names.extend(spec.fields.values().map(|p| p.dotted())); + } + PlanNode::With { columns } => { + for col in columns { + collect_with_columns(&col.expr, names); + } + } + PlanNode::Limit { .. } => {} + PlanNode::Dedupe { keys } | PlanNode::Distinct { keys } => { + names.extend(keys.iter().map(|k| k.dotted())); + } + } +} + +fn collect_agg_columns(agg: &TypedAggregate, names: &mut Vec) { + match agg { + TypedAggregate::Count { .. } => {} + TypedAggregate::Numeric { field, .. } | TypedAggregate::MoneySum { field, .. } => { + names.push(field.dotted()); + } + } +} + +fn collect_with_columns(expr: &WithExpr, names: &mut Vec) { + match expr { + WithExpr::Field(p) | WithExpr::Len { field: p } => { + names.push(p.dotted()); + } + WithExpr::Literal(_) | WithExpr::Now => {} + WithExpr::Arith { lhs, rhs, .. } => { + collect_with_columns(lhs, names); + collect_with_columns(rhs, names); + } + WithExpr::When { + lhs, + rhs, + then, + else_, + .. + } => { + collect_with_columns(lhs, names); + collect_with_columns(rhs, names); + collect_with_columns(then, names); + collect_with_columns(else_, names); + } + } +} + +fn finalize_money_sums(state: &mut FrameState) -> PolarsResult<()> { + let names = std::mem::take(&mut state.money_sum_names); + for name in names { + let n_col = format!("__ccy_n_{name}"); + let c_col = format!("__ccy_{name}"); + let n_unique = state.df.column(&n_col)?; + let ccys = state.df.column(&c_col)?; + for i in 0..state.df.height() { + let n = match n_unique.get(i)? { + AnyValue::UInt32(n) => n as u64, + AnyValue::UInt64(n) => n, + AnyValue::Int64(n) if n >= 0 => n as u64, + AnyValue::Int32(n) if n >= 0 => n as u64, + AnyValue::Null => 0, + other => { + return Err(PolarsError::ComputeError( + format!("unexpected currency-count dtype {other:?}").into(), + )) + } + }; + if n > 1 { + let left = match ccys.get(i)? { + AnyValue::String(s) => s.to_string(), + AnyValue::StringOwned(s) => s.as_str().to_string(), + _ => "left".into(), + }; + return Err(PolarsError::ComputeError( + format!("cannot compare money in {left} to money in another currency").into(), + )); + } + } + let amounts = state.df.column(&name)?; + let mut encoded: Vec> = Vec::with_capacity(state.df.height()); + for i in 0..state.df.height() { + let amount = any_amount_string(amounts.get(i)?)?; + let ccy = match ccys.get(i)? { + AnyValue::String(s) => s.to_string(), + AnyValue::StringOwned(s) => s.as_str().to_string(), + AnyValue::Null => String::new(), + other => other.to_string(), + }; + let mut map = serde_json::Map::new(); + map.insert("__plasm_money".into(), serde_json::Value::String(amount)); + if !ccy.is_empty() { + map.insert("currency".into(), serde_json::Value::String(ccy)); + } + encoded.push(Some(serde_json::Value::Object(map).to_string())); + } + let series = Series::new(PlSmallStr::from_str(&name), encoded); + state.df.with_column(series)?; + let _ = state.df.drop_in_place(&n_col); + let _ = state.df.drop_in_place(&c_col); + state.kinds.insert(name, ColKind::Json); + } + Ok(()) +} + +fn any_amount_string(v: AnyValue<'_>) -> PolarsResult { + Ok(match v { + AnyValue::Decimal(unscaled, scale) => format_decimal_i128(unscaled, scale), + AnyValue::Float64(f) => trim_float(f), + AnyValue::Float32(f) => trim_float(f as f64), + AnyValue::Int64(i) => i.to_string(), + AnyValue::Int32(i) => i.to_string(), + AnyValue::String(s) => s.to_string(), + AnyValue::StringOwned(s) => s.as_str().to_string(), + AnyValue::Null => "0".into(), + other => { + return Err(PolarsError::ComputeError( + format!("cannot encode money amount from {other:?}").into(), + )) + } + }) +} + +fn format_decimal_i128(unscaled: i128, scale: usize) -> String { + Decimal::from_i128_with_scale(unscaled, scale as u32) + .normalize() + .to_string() +} + +fn trim_float(f: f64) -> String { + let d = Decimal::from_f64_retain(f) + .unwrap_or(Decimal::ZERO) + .normalize(); + d.to_string() +} + +fn apply_node( + lf: LazyFrame, + node: &PlanNode, + state: &mut FrameState, + now: DateTime, +) -> PolarsResult { + match node { + PlanNode::Filter(filter) => { + let mut e = lit(true); + for p in filter.predicates() { + e = e.and(pred_expr(p)?); + } + Ok(lf.filter(e)) + } + PlanNode::Sort { key, descending } => Ok(lf.sort( + [key.dotted()], + SortMultipleOptions::default() + .with_order_descending(*descending) + .with_nulls_last(true) + .with_maintain_order(true), + )), + PlanNode::Limit { count } => Ok(lf.slice(0, count.get() as u32)), + PlanNode::Dedupe { keys } | PlanNode::Distinct { keys } => { + let subset: Option> = if keys.is_empty() { + None + } else { + Some( + keys.iter() + .map(|k| PlSmallStr::from_string(k.dotted())) + .collect(), + ) + }; + Ok(lf.unique_stable(subset, UniqueKeepStrategy::First)) + } + PlanNode::Project(spec) => { + let mut exprs = vec![col(IDX_COL)]; + let mut output_columns = Vec::new(); + for (name, path) in &spec.fields { + exprs.push(col_expr(path).alias(name.as_str())); + let kind = state + .kinds + .get(&path.dotted()) + .copied() + .unwrap_or(ColKind::Json); + output_columns.push((name.as_str().to_string(), kind)); + } + state.replace_output_columns(output_columns); + Ok(lf.select(exprs)) + } + PlanNode::With { columns } => { + let mut exprs = Vec::new(); + for col_def in columns { + let e = with_expr(&col_def.expr, state, now)?; + let name = col_def.name.as_str(); + let kind = infer_with_kind(&col_def.expr, state); + state.add_output_column(name.to_string(), kind); + exprs.push(e.alias(name)); + } + Ok(lf.with_columns(exprs)) + } + PlanNode::GroupBy { keys, aggs } => group_by_lf(lf, keys, aggs, state, true), + PlanNode::Aggregate { aggs } => group_by_lf(lf, &[], aggs, state, false), + } +} + +fn group_by_lf( + lf: LazyFrame, + keys: &[FieldPath], + aggs: &[TypedAggregate], + state: &mut FrameState, + grouped: bool, +) -> PolarsResult { + let mut agg_exprs = Vec::new(); + let mut visible = Vec::new(); + for k in keys { + visible.push(k.dotted()); + } + for agg in aggs { + match agg { + TypedAggregate::Count { name } => { + agg_exprs.push(len().alias(name.as_str())); + visible.push(name.as_str().to_string()); + state.kinds.insert(name.as_str().to_string(), ColKind::Int); + } + TypedAggregate::Numeric { name, fn_, field } => { + if *fn_ == plasm_core::row_plan::NumericAgg::Sum + && state.kinds.get(&field.dotted()) == Some(&ColKind::Money) + { + push_money_sum(&mut agg_exprs, &mut visible, state, name.as_str(), field); + } else { + let c = col_expr(field).cast(DataType::Float64); + let e = match fn_ { + plasm_core::row_plan::NumericAgg::Sum => c.sum(), + plasm_core::row_plan::NumericAgg::Avg => c.mean(), + plasm_core::row_plan::NumericAgg::Min => c.min(), + plasm_core::row_plan::NumericAgg::Max => c.max(), + plasm_core::row_plan::NumericAgg::First => c.first(), + plasm_core::row_plan::NumericAgg::Last => c.last(), + }; + agg_exprs.push(e.alias(name.as_str())); + visible.push(name.as_str().to_string()); + state + .kinds + .insert(name.as_str().to_string(), ColKind::Float); + } + } + TypedAggregate::MoneySum { name, field, .. } => { + push_money_sum(&mut agg_exprs, &mut visible, state, name.as_str(), field); + } + } + } + state.visible = visible; + if grouped { + Ok(lf + .group_by(keys.iter().map(|k| col(k.dotted())).collect::>()) + .agg(agg_exprs)) + } else { + Ok(lf.select(agg_exprs)) + } +} + +fn push_money_sum( + agg_exprs: &mut Vec, + visible: &mut Vec, + state: &mut FrameState, + name: &str, + field: &FieldPath, +) { + let amount = col_expr(field) + .struct_() + .field_by_name(MONEY_AMOUNT) + .cast(DataType::Decimal(Some(38), Some(8))); + let ccy = col_expr(field).struct_().field_by_name(MONEY_CCY); + agg_exprs.push(ccy.clone().n_unique().alias(format!("__ccy_n_{name}"))); + agg_exprs.push(ccy.first().alias(format!("__ccy_{name}"))); + agg_exprs.push(amount.sum().alias(name)); + visible.push(name.to_string()); + state.kinds.insert(name.to_string(), ColKind::Money); + state.register_money_sum(name); +} + +fn pred_expr(p: &PlanPredicate) -> PolarsResult { + let lhs = col_expr(&p.field_path); + let rhs = data_lit(&p.value)?; + Ok(predicate_op_expr(p.op, lhs, rhs)) +} + +fn data_lit(v: &PlasmDataValue) -> PolarsResult { + match v { + PlasmDataValue::Literal { value } => json_lit(value), + PlasmDataValue::Array { items } => { + let lits: Result, _> = items.iter().map(data_lit).collect(); + Ok(concat_list(lits?)?) + } + other => Err(PolarsError::ComputeError( + format!("unsupported row-filter value {other:?}").into(), + )), + } +} + +fn json_lit(v: &serde_json::Value) -> PolarsResult { + Ok(match v { + serde_json::Value::Null => lit(NULL), + serde_json::Value::Bool(b) => lit(*b), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + lit(i) + } else if let Some(f) = n.as_f64() { + lit(f) + } else { + lit(n.to_string()) + } + } + serde_json::Value::String(s) => lit(s.as_str()), + serde_json::Value::Array(items) => { + let lits: Result, _> = items.iter().map(json_lit).collect(); + concat_list(lits?)? + } + serde_json::Value::Object(_) => lit(v.to_string()), + }) +} + +fn with_expr(expr: &WithExpr, state: &FrameState, now: DateTime) -> PolarsResult { + match expr { + WithExpr::Field(path) => Ok(col_expr(path)), + WithExpr::Now => Ok(lit(now.to_rfc3339())), + WithExpr::Literal(litv) => Ok(match litv { + WithLiteral::Null => lit(NULL), + WithLiteral::Bool(b) => lit(*b), + WithLiteral::Integer(i) => lit(*i), + WithLiteral::Number(s) => { + if let Ok(i) = s.parse::() { + lit(i) + } else if let Ok(f) = s.parse::() { + lit(f) + } else { + lit(s.as_str()) + } + } + WithLiteral::String(s) => lit(s.as_str()), + }), + WithExpr::Arith { op, lhs, rhs } + if *op == ArithOp::Sub && is_temporal_sub(lhs, rhs, state) => + { + temporal_sub_days(now, lhs, rhs) + } + WithExpr::Arith { op, lhs, rhs } => { + let l_kind = infer_with_kind(lhs, state); + let r_kind = infer_with_kind(rhs, state); + let l = with_expr(lhs, state, now)?; + let r = with_expr(rhs, state, now)?; + arith_expr(*op, l, r, l_kind, r_kind) + } + WithExpr::Len { field } => Ok(col_expr(field) + .cast(DataType::String) + .str() + .len_chars() + .cast(DataType::Int64)), + WithExpr::When { + lhs, + op, + rhs, + then, + else_, + } => { + let l = with_expr(lhs, state, now)?; + let r = with_expr(rhs, state, now)?; + Ok(when(cmp_exprs(*op, l, r)) + .then(with_expr(then, state, now)?) + .otherwise(with_expr(else_, state, now)?)) + } + } +} + +fn is_now(expr: &WithExpr) -> bool { + matches!(expr, WithExpr::Now) +} + +fn is_temporal_operand(expr: &WithExpr, state: &FrameState) -> bool { + match expr { + WithExpr::Now => true, + WithExpr::Literal(WithLiteral::String(_)) => true, + WithExpr::Field(p) => matches!( + state.kinds.get(&p.dotted()), + Some(ColKind::Str | ColKind::Temporal) + ), + _ => false, + } +} + +fn is_temporal_sub(lhs: &WithExpr, rhs: &WithExpr, state: &FrameState) -> bool { + is_now(lhs) + || is_now(rhs) + || (is_temporal_operand(lhs, state) && is_temporal_operand(rhs, state)) +} + +fn utc_from_raw(raw: &str) -> Option> { + normalize_temporal_value( + plasm_core::Value::String(raw.to_string()), + TemporalWireFormat::Rfc3339, + ) + .ok() + .and_then(|v| match v { + plasm_core::Value::String(iso) => DateTime::parse_from_rfc3339(&iso) + .ok() + .map(|dt| dt.with_timezone(&Utc)), + _ => None, + }) +} + +const MS_PER_DAY: i64 = 86_400_000; + +fn col_to_epoch_millis(field: &FieldPath) -> Expr { + col_expr(field).map( + move |s| { + let out: Vec> = match s.dtype() { + DataType::String => s + .str() + .map(|ca| { + ca.into_iter() + .map(|opt| { + opt.and_then(|raw| { + utc_from_raw(raw).map(|dt| dt.timestamp_millis()) + }) + }) + .collect() + }) + .unwrap_or_default(), + _ => vec![None; s.len()], + }; + Ok(Some(Column::new(s.name().clone(), out))) + }, + GetOutput::from_type(DataType::Int64), + ) +} + +fn temporal_millis_expr(expr: &WithExpr, now: DateTime) -> PolarsResult { + match expr { + WithExpr::Now => Ok(lit(now.timestamp_millis())), + WithExpr::Field(field) => Ok(col_to_epoch_millis(field)), + WithExpr::Literal(WithLiteral::String(s)) => Ok(match utc_from_raw(s) { + Some(dt) => lit(dt.timestamp_millis()), + None => lit(NULL).cast(DataType::Int64), + }), + _ => Err(PolarsError::ComputeError( + "temporal subtraction requires temporal fields or `now`".into(), + )), + } +} + +fn temporal_sub_days(now: DateTime, lhs: &WithExpr, rhs: &WithExpr) -> PolarsResult { + let l = temporal_millis_expr(lhs, now)?; + let r = temporal_millis_expr(rhs, now)?; + Ok((l - r) / lit(MS_PER_DAY)) +} + +fn cmp_exprs(op: PlanPredicateOp, l: Expr, r: Expr) -> Expr { + predicate_op_expr(op, l, r) +} + +fn predicate_op_expr(op: PlanPredicateOp, l: Expr, r: Expr) -> Expr { + match op { + PlanPredicateOp::Eq => l.eq(r), + PlanPredicateOp::Ne => l.neq(r), + PlanPredicateOp::Lt => l.lt(r), + PlanPredicateOp::Lte => l.lt_eq(r), + PlanPredicateOp::Gt => l.gt(r), + PlanPredicateOp::Gte => l.gt_eq(r), + PlanPredicateOp::Contains => l.cast(DataType::String).str().contains(r, false), + PlanPredicateOp::In => l.is_in(r), + PlanPredicateOp::Exists => l.is_not_null(), + } +} + +fn arith_expr( + op: ArithOp, + l: Expr, + r: Expr, + l_kind: ColKind, + r_kind: ColKind, +) -> PolarsResult { + let money_l = l_kind == ColKind::Money; + let money_r = r_kind == ColKind::Money; + if !money_l && !money_r { + let string_add = op == ArithOp::Add + && (l_kind == ColKind::Str || r_kind == ColKind::Str) + && l_kind != ColKind::Temporal + && r_kind != ColKind::Temporal; + if string_add { + return Ok(l.cast(DataType::String) + r.cast(DataType::String)); + } + let coerce = op == ArithOp::Div + || matches!(l_kind, ColKind::Str | ColKind::Json | ColKind::Float) + || matches!(r_kind, ColKind::Str | ColKind::Json | ColKind::Float); + let l = if coerce { l.cast(DataType::Float64) } else { l }; + let r = if coerce { r.cast(DataType::Float64) } else { r }; + return Ok(match op { + ArithOp::Add => l + r, + ArithOp::Sub => l - r, + ArithOp::Mul => l * r, + ArithOp::Div => l / r, + }); + } + let l_amt = if money_l { + l.clone() + .struct_() + .field_by_name(MONEY_AMOUNT) + .cast(DataType::Decimal(Some(38), Some(8))) + } else { + l.clone().cast(DataType::Decimal(Some(38), Some(8))) + }; + let r_amt = if money_r { + r.clone() + .struct_() + .field_by_name(MONEY_AMOUNT) + .cast(DataType::Decimal(Some(38), Some(8))) + } else { + r.clone().cast(DataType::Decimal(Some(38), Some(8))) + }; + let amount = match op { + ArithOp::Add => l_amt + r_amt, + ArithOp::Sub => l_amt - r_amt, + ArithOp::Mul => l_amt * r_amt, + ArithOp::Div => l_amt / r_amt, + }; + let ccy = if money_l { + l.struct_().field_by_name(MONEY_CCY) + } else { + r.struct_().field_by_name(MONEY_CCY) + }; + Ok(as_struct(vec![ + amount.cast(DataType::String).alias(MONEY_AMOUNT), + ccy.alias(MONEY_CCY), + ])) +} + +fn infer_with_kind(expr: &WithExpr, state: &FrameState) -> ColKind { + match expr { + WithExpr::Field(p) => state + .kinds + .get(&p.dotted()) + .copied() + .unwrap_or(ColKind::Json), + WithExpr::Now => ColKind::Temporal, + WithExpr::Literal(WithLiteral::Bool(_)) => ColKind::Bool, + WithExpr::Literal(WithLiteral::Integer(_)) => ColKind::Int, + WithExpr::Literal(WithLiteral::Number(_)) => ColKind::Float, + WithExpr::Literal(WithLiteral::String(_)) => ColKind::Str, + WithExpr::Literal(WithLiteral::Null) => ColKind::Json, + WithExpr::Len { .. } => ColKind::Int, + WithExpr::Arith { op, lhs, rhs } => { + if *op == ArithOp::Sub && is_temporal_sub(lhs, rhs, state) { + return ColKind::Int; + } + let l = infer_with_kind(lhs, state); + let r = infer_with_kind(rhs, state); + if *op == ArithOp::Add + && (l == ColKind::Str || r == ColKind::Str) + && l != ColKind::Temporal + && r != ColKind::Temporal + && l != ColKind::Money + && r != ColKind::Money + { + return ColKind::Str; + } + if l == ColKind::Money || r == ColKind::Money { + ColKind::Money + } else if *op == ArithOp::Div || l == ColKind::Float || r == ColKind::Float { + ColKind::Float + } else { + l + } + } + WithExpr::When { then, .. } => infer_with_kind(then, state), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use plasm_core::parse_with_body; + use std::str::FromStr; + + #[test] + fn filter_sort_limit_roundtrip() { + let rows = vec![ + serde_json::json!({"owner":"alice","score":10}), + serde_json::json!({"owner":"bob","score":30}), + serde_json::json!({"owner":"alice","score":20}), + ]; + let pred = plasm_core::PlanPredicate { + field_path: FieldPath::from_dotted("owner").unwrap(), + op: PlanPredicateOp::Eq, + value: PlasmDataValue::Literal { + value: serde_json::json!("alice"), + }, + }; + let ops = vec![ + ComputeOp::Filter { + predicates: vec![pred], + }, + ComputeOp::Sort { + key: FieldPath::from_dotted("score").unwrap(), + descending: true, + }, + ComputeOp::Limit { count: 1 }, + ]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out.len(), 1); + assert_eq!(out[0]["score"], serde_json::json!(20)); + } + + #[test] + fn sort_orders_multi_digit_numbers_numerically() { + let rows = vec![ + serde_json::json!({"score": 87}), + serde_json::json!({"score": 300}), + ]; + let ops = vec![ComputeOp::Sort { + key: FieldPath::from_dotted("score").unwrap(), + descending: true, + }]; + + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + + assert_eq!(out[0]["score"], serde_json::json!(300)); + assert_eq!(out[1]["score"], serde_json::json!(87)); + } + + #[test] + fn with_mul_adds_column() { + let rows = vec![serde_json::json!({"quantity": 2, "price": 5})]; + let columns = parse_with_body("notional: quantity * price").unwrap(); + let ops = vec![ComputeOp::With { columns }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out[0]["notional"], serde_json::json!(10)); + assert_eq!(out[0]["quantity"], serde_json::json!(2)); + } + + #[test] + fn with_now_minus_field_is_nonnegative_int_days() { + let rows = vec![ + serde_json::json!({"id": "old", "updated_at": "2020-01-01T00:00:00Z"}), + serde_json::json!({"id": "new", "updated_at": "2024-06-01T00:00:00Z"}), + ]; + let columns = parse_with_body("age_days: (now - updated_at)").unwrap(); + let ops = vec![ComputeOp::With { columns }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + let older = out.iter().find(|r| r["id"] == "old").unwrap(); + let newer = out.iter().find(|r| r["id"] == "new").unwrap(); + let age_old = older["age_days"].as_i64().expect("age int"); + let age_new = newer["age_days"].as_i64().expect("age int"); + assert!(age_old >= 0 && age_new >= 0, "ages {age_old} {age_new}"); + assert!( + age_old > age_new, + "older row must have larger age: {age_old} vs {age_new}" + ); + } + + #[test] + fn with_field_minus_field_is_int_days() { + let rows = vec![serde_json::json!({ + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-11T00:00:00Z", + })]; + let columns = parse_with_body("cycle: (updated_at - created_at)").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["cycle"], serde_json::json!(10)); + } + + #[test] + fn with_div_is_float() { + let rows = vec![serde_json::json!({"quantity": 10, "price": 4})]; + let columns = parse_with_body("rate: quantity / price").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["rate"].as_f64().unwrap(), 2.5); + } + + #[test] + fn with_string_plus_concat() { + let rows = vec![serde_json::json!({"first": "al", "last": "ice"})]; + let columns = parse_with_body("name: first + last").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["name"], serde_json::json!("alice")); + } + + #[test] + fn with_when_len_and_temporal_cmp() { + let rows = vec![ + serde_json::json!({ + "title": "", + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-02T00:00:00Z", + }), + serde_json::json!({ + "title": "ok", + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-20T00:00:00Z", + }), + ]; + let columns = parse_with_body( + "blank: when(len(title)=0, 1, 0), long: when(updated_at - created_at > 5, 1, 0)", + ) + .unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["blank"], serde_json::json!(1)); + assert_eq!(out[0]["long"], serde_json::json!(0)); + assert_eq!(out[1]["blank"], serde_json::json!(0)); + assert_eq!(out[1]["long"], serde_json::json!(1)); + } + + #[test] + fn with_when_now_minus_gt() { + let rows = vec![ + serde_json::json!({"id": "old", "updated_at": "2020-01-01T00:00:00Z"}), + serde_json::json!({"id": "future", "updated_at": "2099-01-01T00:00:00Z"}), + ]; + let columns = parse_with_body("stale: when(now - updated_at > 14, 1, 0)").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + let old = out.iter().find(|r| r["id"] == "old").unwrap(); + let future = out.iter().find(|r| r["id"] == "future").unwrap(); + assert_eq!(old["stale"], serde_json::json!(1)); + assert_eq!(future["stale"], serde_json::json!(0)); + } + + #[test] + fn group_by_count() { + let rows = vec![ + serde_json::json!({"owner":"a","score":1}), + serde_json::json!({"owner":"a","score":2}), + serde_json::json!({"owner":"b","score":3}), + ]; + let ops = vec![ComputeOp::GroupBy { + keys: vec![FieldPath::from_dotted("owner").unwrap()], + aggregates: vec![plasm_core::AggregateSpec { + name: plasm_core::OutputName::new("n").unwrap(), + function: plasm_core::AggregateFunction::Count, + field: None, + }], + }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out.len(), 2); + } + + #[test] + fn money_sum_same_currency() { + let rows = vec![ + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"1.50","currency":"USD"}}), + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"2.50","currency":"USD"}}), + serde_json::json!({"symbol":"B","fee":{"__plasm_money":"4.00","currency":"USD"}}), + ]; + let ops = vec![ComputeOp::GroupBy { + keys: vec![FieldPath::from_dotted("symbol").unwrap()], + aggregates: vec![plasm_core::AggregateSpec { + name: plasm_core::OutputName::new("fees").unwrap(), + function: plasm_core::AggregateFunction::Sum, + field: Some(FieldPath::from_dotted("fee").unwrap()), + }], + }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out.len(), 2, "out={out:?}"); + let a = out.iter().find(|r| r["symbol"] == "A").unwrap(); + let got = a["fees"]["__plasm_money"].as_str().expect("money amount"); + assert_eq!( + Decimal::from_str(got).unwrap(), + Decimal::from_str("4.00").unwrap(), + "row={a:?}" + ); + assert_eq!(a["fees"]["currency"], "USD"); + assert!(a.get("__ccy_n").is_none()); + assert!(a.get("__ccy_n_fees").is_none()); + } + + #[test] + fn money_sum_rejects_cross_currency() { + let rows = vec![ + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"1.00","currency":"USD"}}), + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"1.00","currency":"EUR"}}), + ]; + let ops = vec![ComputeOp::GroupBy { + keys: vec![FieldPath::from_dotted("symbol").unwrap()], + aggregates: vec![plasm_core::AggregateSpec { + name: plasm_core::OutputName::new("fees").unwrap(), + function: plasm_core::AggregateFunction::Sum, + field: Some(FieldPath::from_dotted("fee").unwrap()), + }], + }]; + let err = eval_compute_ops(&ops, &rows).unwrap_err(); + assert!( + err.contains("currency") || err.contains("money"), + "expected cross-currency error, got {err}" + ); + } +} diff --git a/crates/plasm-runtime/src/row_compute/json_frame.rs b/crates/plasm-runtime/src/row_compute/json_frame.rs new file mode 100644 index 00000000..5bf29a49 --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/json_frame.rs @@ -0,0 +1,407 @@ +//! JSON object rows ↔ Polars DataFrame. Nested objects stay JSON strings; dotted paths +//! are extra columns used only for FieldPath access. + +use indexmap::IndexMap; +use plasm_core::money::MoneyValue; +use plasm_core::{json_value_to_plasm_value, Value}; +use polars::prelude::*; +use rust_decimal::Decimal; +use std::str::FromStr; + +pub(super) const IDX_COL: &str = "__plasm_idx"; +pub(super) const MONEY_AMOUNT: &str = "__amount"; +pub(super) const MONEY_CCY: &str = "__ccy"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ColKind { + Bool, + Int, + Float, + Str, + Temporal, + Money, + Json, +} + +#[derive(Debug, Clone)] +pub(super) struct FrameState { + pub df: DataFrame, + pub visible: Vec, + pub kinds: IndexMap, + /// Output names of money `sum` aggregates pending reconstruct + currency check. + pub money_sum_names: Vec, +} + +impl FrameState { + pub(super) fn ensure_visible_kind(&mut self, name: String, kind: ColKind) { + if !self.visible.iter().any(|visible| visible == &name) { + self.visible.push(name.clone()); + } + self.kinds.entry(name).or_insert(kind); + } + + pub(super) fn add_output_column(&mut self, name: String, kind: ColKind) { + if !self.visible.iter().any(|visible| visible == &name) { + self.visible.push(name.clone()); + } + self.kinds.insert(name, kind); + } + + pub(super) fn replace_output_columns(&mut self, columns: Vec<(String, ColKind)>) { + self.visible.clear(); + for (name, kind) in columns { + self.visible.push(name.clone()); + self.kinds.insert(name, kind); + } + } + + pub(super) fn register_money_sum(&mut self, name: &str) { + self.money_sum_names.push(name.to_string()); + } +} + +pub(super) fn ingest_json_rows(rows: &[serde_json::Value]) -> PolarsResult { + let mut visible = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for row in rows { + if let serde_json::Value::Object(map) = row { + for k in map.keys() { + if seen.insert(k.clone()) { + visible.push(k.clone()); + } + } + } + } + let mut kinds: IndexMap = IndexMap::new(); + let mut columns: IndexMap>> = IndexMap::new(); + for key in &visible { + columns.insert(key.clone(), Vec::with_capacity(rows.len())); + } + let mut extra: IndexMap>> = IndexMap::new(); + + for (i, row) in rows.iter().enumerate() { + let obj = match row { + serde_json::Value::Object(m) => m, + other => { + let mut m = serde_json::Map::new(); + m.insert("value".into(), other.clone()); + for key in &visible { + let cell = json_to_any(m.get(key).unwrap_or(&serde_json::Value::Null)); + columns.get_mut(key).unwrap().push(cell); + } + flatten_into("", other, &mut extra, i, rows.len()); + continue; + } + }; + for key in &visible { + let v = obj.get(key).unwrap_or(&serde_json::Value::Null); + let cell = json_to_any(v); + let kind = kind_of(&cell); + kinds + .entry(key.clone()) + .and_modify(|k| *k = promote(*k, kind)) + .or_insert(kind); + columns.get_mut(key).unwrap().push(cell); + flatten_into(key, v, &mut extra, i, rows.len()); + } + } + + let mut series: Vec = Vec::new(); + let idx: Vec = (0..rows.len() as u32).collect(); + series.push(Column::new(PlSmallStr::from_static(IDX_COL), idx)); + for (key, vals) in columns { + series.push(series_from_any(key.as_str(), vals, kinds.get(&key).copied())?.into()); + } + for (key, vals) in extra { + if visible.iter().any(|v| v == &key) { + continue; + } + let kind = vals.iter().find_map(|v| { + let k = kind_of(v); + if k == ColKind::Json && matches!(v, AnyValue::Null) { + None + } else { + Some(k) + } + }); + kinds + .entry(key.clone()) + .or_insert(kind.unwrap_or(ColKind::Json)); + series.push(series_from_any(key.as_str(), vals, kinds.get(&key).copied())?.into()); + } + Ok(FrameState { + df: DataFrame::new(series)?, + visible, + kinds, + money_sum_names: Vec::new(), + }) +} + +fn flatten_into( + prefix: &str, + v: &serde_json::Value, + extra: &mut IndexMap>>, + row_i: usize, + n: usize, +) { + let serde_json::Value::Object(map) = v else { + return; + }; + for (k, child) in map { + let path = if prefix.is_empty() { + k.clone() + } else { + format!("{prefix}.{k}") + }; + let slot = extra + .entry(path.clone()) + .or_insert_with(|| vec![AnyValue::Null; n]); + if slot.len() < n { + slot.resize(n, AnyValue::Null); + } + slot[row_i] = json_to_any(child); + flatten_into(&path, child, extra, row_i, n); + } +} + +fn json_to_any(v: &serde_json::Value) -> AnyValue<'static> { + match json_value_to_plasm_value(v) { + Value::Null => AnyValue::Null, + Value::Bool(b) => AnyValue::Boolean(b), + Value::Integer(i) => AnyValue::Int64(i), + Value::Float(f) => AnyValue::Float64(f), + Value::String(s) | Value::PhraseIdent(s) => AnyValue::StringOwned(s.into()), + Value::Money(m) => money_any(&m), + Value::Array(_) | Value::Object(_) | Value::UnionCtor { .. } | Value::PlasmInputRef(_) => { + AnyValue::StringOwned(v.to_string().into()) + } + } +} + +fn money_any(m: &MoneyValue) -> AnyValue<'static> { + let amount = m.amount().to_string(); + let ccy = m.currency().unwrap_or("").to_string(); + AnyValue::StructOwned(Box::new(( + vec![ + AnyValue::StringOwned(amount.into()), + AnyValue::StringOwned(ccy.into()), + ], + vec![ + Field::new(PlSmallStr::from_static(MONEY_AMOUNT), DataType::String), + Field::new(PlSmallStr::from_static(MONEY_CCY), DataType::String), + ], + ))) +} + +fn kind_of(v: &AnyValue<'_>) -> ColKind { + match v { + AnyValue::Null => ColKind::Json, + AnyValue::Boolean(_) => ColKind::Bool, + AnyValue::Int64(_) | AnyValue::Int32(_) | AnyValue::UInt32(_) | AnyValue::UInt64(_) => { + ColKind::Int + } + AnyValue::Float64(_) | AnyValue::Float32(_) => ColKind::Float, + AnyValue::StructOwned(_) | AnyValue::Struct(_, _, _) => ColKind::Money, + AnyValue::String(_) | AnyValue::StringOwned(_) => ColKind::Str, + _ => ColKind::Json, + } +} + +fn promote(a: ColKind, b: ColKind) -> ColKind { + if a == b { + return a; + } + if a == ColKind::Json { + return b; + } + if b == ColKind::Json { + return a; + } + match (a, b) { + (ColKind::Int, ColKind::Float) | (ColKind::Float, ColKind::Int) => ColKind::Float, + (ColKind::Money, _) | (_, ColKind::Money) => ColKind::Money, + _ => ColKind::Str, + } +} + +fn series_from_any( + name: &str, + vals: Vec>, + kind: Option, +) -> PolarsResult { + let name = PlSmallStr::from_str(name); + match kind.unwrap_or(ColKind::Json) { + ColKind::Bool => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Boolean(b) => Some(b), + AnyValue::Null => None, + _ => None, + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Int => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Int64(i) => Some(i), + AnyValue::Int32(i) => Some(i as i64), + AnyValue::UInt32(i) => Some(i as i64), + AnyValue::Null => None, + _ => None, + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Float => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Float64(f) => Some(f), + AnyValue::Int64(i) => Some(i as f64), + AnyValue::Null => None, + _ => None, + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Str | ColKind::Json | ColKind::Temporal => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Null => None, + AnyValue::StringOwned(s) => Some(s.as_str().to_string()), + AnyValue::String(s) => Some(s.to_string()), + AnyValue::Boolean(b) => Some(b.to_string()), + AnyValue::Int64(i) => Some(i.to_string()), + AnyValue::Float64(f) => Some(f.to_string()), + other => Some(other.to_string()), + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Money => Series::from_any_values_and_dtype( + name, + &vals, + &DataType::Struct(vec![ + Field::new(PlSmallStr::from_static(MONEY_AMOUNT), DataType::String), + Field::new(PlSmallStr::from_static(MONEY_CCY), DataType::String), + ]), + true, + ), + } +} + +pub(super) fn collect_json(state: &FrameState) -> PolarsResult> { + let df = &state.df; + let n = df.height(); + let mut out = Vec::with_capacity(n); + for row_idx in 0..n { + let mut map = serde_json::Map::new(); + for key in &state.visible { + if key == IDX_COL { + continue; + } + let Some(s) = df.column(key).ok() else { + continue; + }; + map.insert( + key.clone(), + any_to_json(s.get(row_idx)?, state.kinds.get(key).copied()), + ); + } + out.push(serde_json::Value::Object(map)); + } + Ok(out) +} + +fn any_to_json(v: AnyValue<'_>, kind: Option) -> serde_json::Value { + if kind == Some(ColKind::Money) { + if let Some(m) = money_from_any(&v) { + return money_tagged_json(&m); + } + } + match v { + AnyValue::Null => serde_json::Value::Null, + AnyValue::Boolean(b) => serde_json::Value::Bool(b), + AnyValue::Int64(i) => serde_json::json!(i), + AnyValue::Int32(i) => serde_json::json!(i), + AnyValue::UInt32(i) => serde_json::json!(i), + AnyValue::UInt64(i) => serde_json::json!(i), + AnyValue::Float64(f) => serde_json::Number::from_f64(f) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null), + AnyValue::Float32(f) => serde_json::Number::from_f64(f as f64) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null), + AnyValue::String(s) => parse_json_or_string(s), + AnyValue::StringOwned(s) => parse_json_or_string(s.as_str()), + AnyValue::StructOwned(boxed) => { + if let Some(m) = money_from_struct(&boxed.0, &boxed.1) { + return money_tagged_json(&m); + } + serde_json::Value::Null + } + other => serde_json::Value::String(other.to_string()), + } +} + +fn parse_json_or_string(s: &str) -> serde_json::Value { + let t = s.trim(); + if (t.starts_with('{') && t.ends_with('}')) || (t.starts_with('[') && t.ends_with(']')) { + if let Ok(v) = serde_json::from_str(s) { + return v; + } + } + serde_json::Value::String(s.to_string()) +} + +fn money_tagged_json(m: &MoneyValue) -> serde_json::Value { + let mut map = serde_json::Map::new(); + map.insert( + "__plasm_money".into(), + serde_json::Value::String(m.amount().to_string()), + ); + if let Some(c) = m.currency() { + map.insert("currency".into(), serde_json::Value::String(c.to_string())); + } + serde_json::Value::Object(map) +} + +fn money_from_any(v: &AnyValue<'_>) -> Option { + match v { + AnyValue::StructOwned(boxed) => money_from_struct(&boxed.0, &boxed.1), + _ => None, + } +} + +fn money_from_struct(vals: &[AnyValue<'_>], fields: &[Field]) -> Option { + let mut amount = None; + let mut ccy = None; + for (field, val) in fields.iter().zip(vals.iter()) { + match field.name().as_str() { + MONEY_AMOUNT => { + amount = match val { + AnyValue::String(s) => Decimal::from_str(s).ok(), + AnyValue::StringOwned(s) => Decimal::from_str(s.as_str()).ok(), + _ => None, + } + } + MONEY_CCY => { + ccy = match val { + AnyValue::String(s) if !s.is_empty() => Some(s.to_string()), + AnyValue::StringOwned(s) if !s.is_empty() => Some(s.as_str().to_string()), + _ => None, + } + } + _ => {} + } + } + Some(MoneyValue::new(amount?, ccy)) +} + +pub(super) fn col_expr(path: &plasm_core::FieldPath) -> Expr { + col(path.dotted()) +} diff --git a/crates/plasm-runtime/src/row_compute/mod.rs b/crates/plasm-runtime/src/row_compute/mod.rs new file mode 100644 index 00000000..5d9e0f89 --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/mod.rs @@ -0,0 +1,11 @@ +//! Polars adapter for fused [`plasm_core::RowPlan`] execute. +//! +//! Public types here do not re-export `polars::*`. [`ComputeOp`] stays the hashed constructor; +//! this module is the only physical engine. + +mod adapter; +mod eval; +mod json_frame; + +pub use adapter::PolarsAdapter; +pub use eval::{eval_compute_ops, ComputeEvalOutcome}; diff --git a/doc-site/docs/concepts.md b/doc-site/docs/concepts.md index 510d025e..3138a918 100644 --- a/doc-site/docs/concepts.md +++ b/doc-site/docs/concepts.md @@ -54,7 +54,7 @@ Agents write **Plasm** programs against symbols exposed in **teaching table** in Legacy opaque `p#` tokens for fields/params are **rejected** at parse. -Expressions compose with pipes and postfix transforms. Multi-line payloads use tagged **heredocs** — see the [Language definition](reference/plasm-language-definition.md). +Expressions compose with pipes and postfix transforms (`.filter`, `.with`, `.sort`, `.group_by`, …). Multi-line payloads use tagged **heredocs** — see the [Language definition](reference/plasm-language-definition.md). With the **`plasm`** remote client, the **client owns the monotonic symbol table** locally; the server executes expanded programs over HTTP. See [Remote terminal](reference/plasm-cgs-remote-terminal.md). diff --git a/doc-site/docs/glossary.md b/doc-site/docs/glossary.md index 80c4c7c7..2082ed6c 100644 --- a/doc-site/docs/glossary.md +++ b/doc-site/docs/glossary.md @@ -3,6 +3,8 @@ | Term | Meaning | |------|---------| | **CGS** | Capability Graph Schema — `domain.yaml` semantic model (entities, relations, capabilities; split catalogs use **`values:`** + **`value_ref`**). | +| **`.with`** | Row-compute postfix that adds derived columns per row (`.with{col: expr}`) while preserving entity identity — see [Row compute](reference/plasm-row-compute.md#derived-columns-with). | +| **RowPlan** | Fused execute-time IR for row compute (`plasm_core::row_plan`); Polars-backed evaluation in `plasm_runtime::row_compute`. | | **CML** | Capability Mapping Language — `mappings.yaml` wire templates. | | **teaching table** | Symbol-tuned teaching text (`e#` / `m#` / `r#` plus **wire names** for fields/params; `v#` gloss only) for agents. | | **view** | CGS **`views:`** entry — composed read-only DAG over existing capabilities (not MCP tenant “registry views”). | diff --git a/doc-site/docs/reference/plasm-language-definition.md b/doc-site/docs/reference/plasm-language-definition.md index c7aa9bb5..383fd4f3 100644 --- a/doc-site/docs/reference/plasm-language-definition.md +++ b/doc-site/docs/reference/plasm-language-definition.md @@ -59,7 +59,7 @@ Each entry in `comp.steps` is a tagged serde object (`kind` discriminant). Wire |--------|------|------------| | `invoke` | Read / action / view surface | `plan_kind`, `qualified_entity`, `ir` **xor** `ir_template`, `projection`, `predicates`, `page_size`, `approval` | | `pure` | Literal / artifact data | `data` (`PlasmDataValue`) | -| `map` | Row compute (filter, sort, group, …) | `compute` (`ComputeTemplate`) | +| `map` | Row compute (filter, sort, group, derived columns, …) | `compute` (`ComputeTemplate`) | | `derive` | Per-row map over a source | `derive` (`DeriveTemplate`: `source`, `item_binding`, `inputs`, `value`) | | `flat_map_relation` | Relation fanout (`>>=`) | `relation` (`PlanRelationTraversal`: `source`, `relation`, `target`, `ir`, `binding_proofs`, `materialize`) | | `flat_map_effect` | `for_each` side effects | `source`, `item_binding`, `effect_template`, `projection`, `predicates`, `approval` | @@ -184,7 +184,7 @@ Implementation: unified entity constructor head resolution in [`entity_ref_parse | Fetch filter | `e#{field=…}` | wire is the query param/filter for that entity+capability | | Search filter | `e#~"…"{field=…}` | wire is the **Search**-capability param (homograph-safe vs Create/Update params) | | Relation hop | `receiver.r#` (or wire) | `r#` resolves to a declared relation wire; a filter wire after `.` yields `RelationSegmentWrongRole` except LHS-binding coercion (see [Binding RHS shapes](#binding-rhs-shapes-label--)) | -| Projection / postfix | `[field,…]`, `.sort(field)`, `.group_by(field)`, … | wire names resolve to `rows:` field symbols under the row entity | +| Projection / postfix | `[field,…]`, `.sort(field)`, `.group_by(field)`, `.with{col: expr}`, `.dedupe(…)`, `.distinct(…)`, … | wire names resolve to `rows:` field symbols under the row entity | --- @@ -218,7 +218,7 @@ Cross-binding references (`${stats.content}`, `body=report.content`) are also su ## Invariants -1. **Transforms are core postfix syntax** — `.limit(n)`, `.sort(field, desc)` / `.sort(field,dir)` (whitespace direction sugar accepted), `.filter{…}` / `.filter(…)`, `.aggregate(…)`, `.group_by(field).aggregate(specs)` (primary), `.group_by(field, …)` (comma sugar), `.singleton()`, `.page_size(n)`, bracket projections `[field,…]`, and row-to-text template blocks (`<` on bindings (two uses only):** `source => { k: _.field }` (derive map) or `source => e1(…).update(…)` (for_each). There is no `.derive(…)` surface. Row-to-text uses postfix `rows <`. - **Relation fanout:** `labels = issues.labels` **or** `labels = issues.r#` (opaque relation symbol from teaching TSV) — never `issues => e2.r#` or `source => binding.r#` (compile rejects relation hops on `=>`). A **filter wire after `.`** on a receiver is not a relation hop (use `.r#` or the relation wire). The RHS of `=>` is not `plasm_expr`; entity calls there stringify or fail compile. - **Homograph wires:** query filters and relation hops may share a wire name (e.g. `labels`). In-grammar resolution at the nav position disambiguates: `receiver.r#` / `receiver.labels` is a relation hop; the same wire in `{…}` is a filter/param. Teaching exemplars prefer `.r#` or wire names in relation position. @@ -401,7 +402,14 @@ POSTFIX_OP = "singleton" | "filter" , ( "{" , PRED_LIST , "}" | "(" , PRED_LIST , ")" ) | "aggregate" , "(" , AGG_ARGS , ")" | "group_by" , "(" , GROUP_ARGS , ")" + | "with" , ( "{" , WITH_BODY , "}" | "(" , WITH_BODY , ")" ) + | "dedupe" , [ "(" , FIELD_LIST , ")" ] + | "distinct" , [ "(" , FIELD_LIST , ")" ] | "[" , FIELD_LIST , "]" ; +WITH_BODY = WITH_COLUMN , { "," , WITH_COLUMN } ; +WITH_COLUMN = IDENT , ":" , WITH_EXPR ; +WITH_EXPR = (* v1: field paths, literals, `now`, `len(field)`, `when(cmp, then, else)`, `+ - * /` — see plasm-row-compute.md *) + ; FIELD_LIST = IDENT , { "," , IDENT } ; ``` diff --git a/doc-site/docs/reference/plasm-row-compute.md b/doc-site/docs/reference/plasm-row-compute.md index 49e1e1c1..faaeaa94 100644 --- a/doc-site/docs/reference/plasm-row-compute.md +++ b/doc-site/docs/reference/plasm-row-compute.md @@ -9,7 +9,7 @@ See also [plasm-language-definition.md](plasm-language-definition.md) for full g | Plane | Surface | When to use | |-------|---------|-------------| | **Catalog** | `e1{state="open"}` on a query/get | Reduce data at the API; predicates become query parameters or CML filters. | -| **Row** | `rows.filter{owner="alice"}` or `rows.filter(owner="alice")` | Filter, sort, group, or aggregate rows already fetched into the session artifact. | +| **Row** | `rows.filter{owner="alice"}` or `rows.filter(owner="alice")` or `rows.with{age: (now - updated_at)}` | Filter, derive columns, sort, group, or aggregate rows already fetched into the session artifact. | Use catalog filters when the API supports them and you want fewer round-trips. Use row filters when refining a binding, combining results from multiple steps, or when the field is not a query parameter. @@ -45,17 +45,47 @@ by_team = LangItem.group_by(owner, team, n=count, total=sum(score)) **aggregate** without a key applies functions over all rows: `all = items.aggregate(n=count)`. +## Derived columns (`.with`) + +Add computed columns to each row while keeping the upstream **entity identity** (relation-dot continuation still works on the binding): + +```text +stale = issues.with{age_days: (now - updated_at)} +boosted = items.with{boost: score * 2} +tagged = items.with{tag: owner + owner} +labeled = items.with{label: when(len(owner)>0, owner, title)} +``` + +**Surface:** `.with{col: expr, …}` or `.with(col: expr, …)` — comma-separated `name: expr` pairs inside the braces/parens. Column names are output labels (wire-style identifiers); expressions reference catalog **field wire names** on the current row. + +**Expression language (v1):** + +| Form | Meaning | +|------|---------| +| `field` / `parent.child` | Field path on the row | +| `null`, `true`, `false`, integer, float, `"text"` | Literals | +| `now` | Catalog-plane UTC clock (not a field lookup — a catalog field named `now` is shadowed) | +| `a + b`, `a - b`, `a * b`, `a / b` | Arithmetic (`+` also concatenates strings) | +| `len(field)` | String length | +| `when(lhs op rhs, then, else)` | Conditional; `op` is `=`, `!=`, `>`, `<`, `>=`, `<=` | + +**Temporal subtraction:** `(now - updated_at)` or `(updated_at - created_at)` yields a non-negative **integer day count** when operands are temporal fields or `now`. Use these in filters or further `.with` columns (e.g. `when(now - updated_at > 14, 1, 0)`). + +**Money:** `*` / `/` / `+` / `-` on money columns follow catalog money typing (same-currency rules; cross-currency arithmetic fails at runtime). + +**Disambiguation:** `.with{` / `.with(` is row compute. Identifiers such as `.join(…)` or `.open(…)` without a leading row-compute verb are **not** postfix operators — they remain path/relation surface and fail row-compute lowering. + ## Chaining order Postfix applies left-to-right on the written expression (`a.limit(10).sort(x)` → sort after limit). Recommended SQL mental model: ```text -source → .filter{…} → .group_by(…) → .sort(…) → .limit(n) → [fields] → < { … }` | **Derive map** over rows — not a relation hop | See [plasm-language-definition.md](plasm-language-definition.md#binding-rhs-shapes-label). **`=>`** is only for derive maps and `for_each` on bindings; relation hops use `.r#`/wire, not `=>`. @@ -73,6 +104,11 @@ See [plasm-language-definition.md](plasm-language-definition.md#binding-rhs-shap - OR/NOT in row filters; `.having{…}`; `.derive()` postfix. - HTTP push-down of row filters (optimizer may add later without changing surface meaning). - `rows{…}` as a row-local filter shorthand. +- Surface `join` / equi-join between bindings (row pipeline rejects join-from-surface). + +## Execution engine + +Row compute lowers fused [`ComputeOp`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/plasm_monad/payload/compute.rs) chains to a [`RowPlan`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/row_plan/plan.rs) IR in `plasm_core::row_plan`, then executes through a Polars-backed adapter in `plasm_runtime::row_compute`. Collect barriers (program return, paging, invoke-arg holes, render) are the only legal materialization points — render and derive remain outside the fused pipeline. ## Federation