fix: preserve multiline source expression whitespace

mdl-examples/bug-tests/322-multiline-source-expression-whitespace.mdl

@@ -0,0 +1,61 @@
+-- ============================================================================
+-- Bug #322: Multiline source expression whitespace lost on roundtrip
+-- ============================================================================
+--
+-- Symptom (before fix):
+--   The visitor rebuilt expression strings from parsed expression nodes,
+--   so original line breaks and whitespace inside DECLARE initial values
+--   and LOG template parameters were normalized away. A multiline
+--   `declare $X string = ...` statement that authors had carefully
+--   formatted across several lines came back as a single very long line
+--   after describe → exec → describe. Inter-parameter blank lines in
+--   `LOG ... WITH (...)` were similarly collapsed, making real-world
+--   microflows non-fixpoint and producing noisy `mxcli diff-local` output.
+--
+-- After fix:
+--   Added a `SourceExpr` AST node that wraps an expression with its
+--   original source text. The visitor uses `buildSourceExpression` for
+--   source-sensitive declare/log/while expressions, and the executor
+--   serializes `SourceExpr` through to MDL output so the original
+--   whitespace and line breaks survive every roundtrip.
+--
+-- Usage:
+--   mxcli exec mdl-examples/bug-tests/322-multiline-source-expression-whitespace.mdl -p app.mpr
+--   mxcli -p app.mpr -c "describe microflow BugTest322.MF_MultilineDeclare"
+--   mxcli -p app.mpr -c "describe microflow BugTest322.MF_MultilineLogTemplate"
+--   The describe outputs must keep the multiline shape and must be
+--   fixpoints under describe → exec → describe.
+-- ============================================================================
+
+create module BugTest322;
+
+-- DECLARE initial value spread across several lines with leading `+`. The
+-- describer must preserve the line breaks rather than collapse them onto
+-- one line.
+create microflow BugTest322.MF_MultilineDeclare (
+  $Page: integer,
+  $Token: string
+)
+returns string as $Endpoint
+begin
+  declare $Endpoint string = '/api/v1'
++ '/items?page=' + toString($Page)
++ '&token=' + $Token;
+
+  return $Endpoint;
+end;
+/
+
+-- LOG template parameters separated by a blank line. The newline-only
+-- whitespace between `{1} = toString($Count)` and the next parameter must
+-- survive the roundtrip.
+create microflow BugTest322.MF_MultilineLogTemplate (
+  $Count: integer,
+  $Endpoint: string
+)
+begin
+  log info node 'BugTest322' 'Processed {1} items for {2}' with ({1} = toString($Count)
+
+, {2} = $Endpoint);
+end;
+/

mdl/ast/ast_expression.go

@@ -125,6 +125,15 @@ type IfThenElseExpr struct {
 
 func (e *IfThenElseExpr) isExpression() {}
 
+// SourceExpr preserves original expression source text while keeping the parsed
+// expression tree available for callers that need semantic inspection.
+type SourceExpr struct {
+	Expression Expression
+	Source     string
+}
+
+func (e *SourceExpr) isExpression() {}
+
 // ============================================================================
 // XPath-Specific Expression Types
 // ============================================================================

mdl/executor/cmd_microflows_builder.go

@@ -266,6 +266,14 @@ func (fb *flowBuilder) resolveAssociationPaths(expr ast.Expression) ast.Expressi
 			ThenExpr:  fb.resolveAssociationPaths(e.ThenExpr),
 			ElseExpr:  fb.resolveAssociationPaths(e.ElseExpr),
 		}
+	case *ast.SourceExpr:
+		if e.Source != "" {
+			return e
+		}
+		return &ast.SourceExpr{
+			Expression: fb.resolveAssociationPaths(e.Expression),
+			Source:     e.Source,
+		}
 	default:
 		return expr
 	}

mdl/executor/cmd_microflows_helpers.go

@@ -321,6 +321,11 @@ func expressionToString(expr ast.Expression) string {
 		thenStr := expressionToString(e.ThenExpr)
 		elseStr := expressionToString(e.ElseExpr)
 		return "if " + cond + " then " + thenStr + " else " + elseStr
+	case *ast.SourceExpr:
+		if e.Source != "" {
+			return e.Source
+		}
+		return expressionToString(e.Expression)
 	default:
 		return ""
 	}
@@ -373,6 +378,11 @@ func expressionToXPath(expr ast.Expression) string {
 		return expressionToString(expr)
 	case *ast.QualifiedNameExpr:
 		return qualifiedNameToXPath(e)
+	case *ast.SourceExpr:
+		if e.Source != "" {
+			return e.Source
+		}
+		return expressionToXPath(e.Expression)
 	default:
 		// For all other expression types, the standard serialization is correct
 		return expressionToString(expr)

mdl/visitor/visitor_microflow_actions.go

@@ -51,10 +51,10 @@ func buildLogStatement(ctx parser.ILogStatementContext) *ast.LogStmt {
 	// LOG always has a message expression; when NODE is present the first
 	// expression is the node and the second is the message.
 	if logCtx.NODE() != nil && len(exprs) > 1 {
-		stmt.Node = buildExpression(exprs[0])
-		stmt.Message = buildExpression(exprs[1])
+		stmt.Node = buildSourceExpression(exprs[0])
+		stmt.Message = buildSourceExpression(exprs[1])
 	} else if len(exprs) > 0 {
-		stmt.Message = buildExpression(exprs[0])
+		stmt.Message = buildSourceExpression(exprs[0])
 	}
 
 	// Parse template parameters: WITH ({1} = expr, {2} = expr, ...)
@@ -79,7 +79,8 @@ func buildTemplateParams(ctx parser.ITemplateParamsContext) []ast.TemplateParam
 	var result []ast.TemplateParam
 
 	// Handle WITH ({1} = expr, {2} = expr, ...) syntax
-	for _, param := range paramsCtx.AllTemplateParam() {
+	allParams := paramsCtx.AllTemplateParam()
+	for i, param := range allParams {
 		paramCtx := param.(*parser.TemplateParamContext)
 		indexStr := paramCtx.NUMBER_LITERAL().GetText()
 		index, _ := strconv.Atoi(indexStr)
@@ -89,7 +90,8 @@ func buildTemplateParams(ctx parser.ITemplateParamsContext) []ast.TemplateParam
 
 		// Parse the expression and check for data source attribute reference
 		if exprCtx := paramCtx.Expression(); exprCtx != nil {
-			expr := buildExpression(exprCtx)
+			expr := buildSourceExpression(exprCtx)
+			expr = appendTemplateParamTrailingWhitespace(paramsCtx, allParams, i, exprCtx, expr)
 			tp.Value = expr
 
 			// Check if this is a $Widget.Attr pattern (AttributePathExpr with Path)
@@ -117,6 +119,70 @@ func buildTemplateParams(ctx parser.ITemplateParamsContext) []ast.TemplateParam
 	return result
 }
 
+func appendTemplateParamTrailingWhitespace(
+	paramsCtx *parser.TemplateParamsContext,
+	allParams []parser.ITemplateParamContext,
+	index int,
+	exprCtx parser.IExpressionContext,
+	expr ast.Expression,
+) ast.Expression {
+	trailing := templateParamTrailingWhitespace(paramsCtx, allParams, index, exprCtx)
+	if trailing == "" || !strings.ContainsAny(trailing, "\r\n") {
+		return expr
+	}
+
+	source := strings.TrimSpace(extractOriginalText(exprCtx.(antlr.ParserRuleContext)))
+	innerExpr := expr
+	if sourceExpr, ok := expr.(*ast.SourceExpr); ok {
+		source = sourceExpr.Source
+		innerExpr = sourceExpr.Expression
+	}
+	return &ast.SourceExpr{Expression: innerExpr, Source: source + trailing}
+}
+
+func templateParamTrailingWhitespace(
+	paramsCtx *parser.TemplateParamsContext,
+	allParams []parser.ITemplateParamContext,
+	index int,
+	exprCtx parser.IExpressionContext,
+) string {
+	exprRule, ok := exprCtx.(antlr.ParserRuleContext)
+	if !ok || exprRule.GetStop() == nil {
+		return ""
+	}
+	input := exprRule.GetStop().GetInputStream()
+	if input == nil {
+		return ""
+	}
+
+	start := exprRule.GetStop().GetStop() + 1
+	end := -1
+	delimiter := byte(')')
+	if index+1 < len(allParams) {
+		nextParam := allParams[index+1].(antlr.ParserRuleContext)
+		end = nextParam.GetStart().GetStart() - 1
+		delimiter = ','
+	} else if paramsCtx.GetStop() != nil {
+		end = paramsCtx.GetStop().GetStart() - 1
+	}
+	if start < 0 || end < start {
+		return ""
+	}
+
+	gap := input.GetText(start, end)
+	if delimiter == ',' {
+		comma := strings.IndexByte(gap, ',')
+		if comma == -1 {
+			return ""
+		}
+		gap = gap[:comma]
+	}
+	if strings.TrimSpace(gap) != "" {
+		return ""
+	}
+	return gap
+}
+
 // buildCallMicroflowStatement converts CALL MICROFLOW statement context to CallMicroflowStmt.
 // Grammar: (VARIABLE EQUALS)? CALL MICROFLOW qualifiedName LPAREN callArgumentList? RPAREN
 func buildCallMicroflowStatement(ctx parser.ICallMicroflowStatementContext) *ast.CallMicroflowStmt {

mdl/visitor/visitor_microflow_statements.go

@@ -5,6 +5,7 @@ package visitor
 import (
 	"strings"
 
+	"github.com/antlr4-go/antlr/v4"
 	"github.com/mendixlabs/mxcli/mdl/ast"
 	"github.com/mendixlabs/mxcli/mdl/grammar/parser"
 )
@@ -494,7 +495,7 @@ func buildDeclareStatement(ctx parser.IDeclareStatementContext) *ast.DeclareStmt
 
 	// Get optional initial value
 	if expr := declCtx.Expression(); expr != nil {
-		stmt.InitialValue = buildExpression(expr)
+		stmt.InitialValue = buildSourceExpression(expr)
 	}
 
 	return stmt
@@ -1068,7 +1069,7 @@ func buildWhileStatement(ctx parser.IWhileStatementContext) *ast.WhileStmt {
 
 	// Get condition expression
 	if expr := wsCtx.Expression(); expr != nil {
-		stmt.Condition = buildExpression(expr)
+		stmt.Condition = buildSourceExpression(expr)
 	}
 
 	// Get body
@@ -1079,6 +1080,51 @@ func buildWhileStatement(ctx parser.IWhileStatementContext) *ast.WhileStmt {
 	return stmt
 }
 
+func buildSourceExpression(ctx parser.IExpressionContext) ast.Expression {
+	if ctx == nil {
+		return nil
+	}
+	expr := buildExpression(ctx)
+	if prc, ok := ctx.(antlr.ParserRuleContext); ok {
+		if source := strings.TrimSpace(extractOriginalText(prc)); source != "" {
+			if shouldPreserveExpressionSource(source) {
+				return &ast.SourceExpr{Expression: expr, Source: source}
+			}
+		}
+	}
+	return expr
+}
+
+func shouldPreserveExpressionSource(source string) bool {
+	if strings.ContainsAny(source, "\r\n") {
+		return true
+	}
+	inString := false
+	for i := 0; i < len(source); i++ {
+		if source[i] == '\'' {
+			if inString && i+1 < len(source) && source[i+1] == '\'' {
+				i++
+				continue
+			}
+			inString = !inString
+			continue
+		}
+		if inString {
+			continue
+		}
+		switch source[i] {
+		case '=', '!', '<', '>', '+', '-', '*', ':':
+			if i > 0 && source[i-1] != ' ' && source[i-1] != '\t' {
+				return true
+			}
+			if i+1 < len(source) && source[i+1] != ' ' && source[i+1] != '\t' && source[i+1] != '=' {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // buildReturnStatement converts RETURN statement context to ReturnStmt.
 func buildReturnStatement(ctx parser.IReturnStatementContext) *ast.ReturnStmt {
 	if ctx == nil {

mdl/visitor/visitor_test.go

@@ -1589,3 +1589,63 @@ func TestCalculatedAttributeOnNonPersistentEntity(t *testing.T) {
 		t.Error("Value attribute should be calculated")
 	}
 }
+
+func TestDeclareAndLogTemplatePreserveMultilineSourceWhitespace(t *testing.T) {
+	input := `CREATE MICROFLOW Synthetic.Check ()
+RETURNS Boolean AS $Success
+BEGIN
+  DECLARE $Endpoint String = @Synthetic.Endpoint
++ '/items?page=' + toString($Page)
++ '&token=' + (if $Token!=empty then $Token/Value
+else
+'');
+  LOG INFO NODE 'SyntheticLog' 'Processed {1} items for {2}' WITH ({1} = toString($Count)
+
+, {2} = $Endpoint);
+  RETURN true;
+END;`
+
+	prog, errs := Build(input)
+	if len(errs) > 0 {
+		t.Fatalf("unexpected parse errors: %v", errs)
+	}
+
+	stmt := prog.Statements[0].(*ast.CreateMicroflowStmt)
+	decl, ok := stmt.Body[0].(*ast.DeclareStmt)
+	if !ok {
+		t.Fatalf("Expected DeclareStmt, got %T", stmt.Body[0])
+	}
+	logStmt, ok := stmt.Body[1].(*ast.LogStmt)
+	if !ok {
+		t.Fatalf("Expected LogStmt, got %T", stmt.Body[1])
+	}
+
+	declSource, ok := decl.InitialValue.(*ast.SourceExpr)
+	if !ok {
+		t.Fatalf("Expected SourceExpr declare value, got %T", decl.InitialValue)
+	}
+	wantDecl := "@Synthetic.Endpoint\n+ '/items?page=' + toString($Page)\n+ '&token=' + (if $Token!=empty then $Token/Value\nelse\n'')"
+	if declSource.Source != wantDecl {
+		t.Fatalf("declare source = %q, want %q", declSource.Source, wantDecl)
+	}
+
+	firstParam, ok := logStmt.Template[0].Value.(*ast.SourceExpr)
+	if !ok {
+		t.Fatalf("Expected SourceExpr first log template param, got %T", logStmt.Template[0].Value)
+	}
+	if firstParam.Source != "toString($Count)\n\n" {
+		t.Fatalf("template param source = %q, want trailing blank line", firstParam.Source)
+	}
+}
+
+func TestShouldPreserveExpressionSourceIgnoresStringLiteralPunctuation(t *testing.T) {
+	if shouldPreserveExpressionSource("'Processed {1} items!'") {
+		t.Fatal("plain string literal punctuation should not force SourceExpr preservation")
+	}
+	if shouldPreserveExpressionSource("'Owner''s item: {1}'") {
+		t.Fatal("escaped quotes and colon inside a string literal should not force SourceExpr preservation")
+	}
+	if !shouldPreserveExpressionSource("$Token!=empty") {
+		t.Fatal("compact operators outside string literals should preserve source")
+	}
+}