From 32170d86b63ec1b4662621eef6733eb54b97efe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Kr=C3=BCger?= Date: Tue, 29 Sep 2026 14:56:57 +0200 Subject: [PATCH 1/2] Use invariant numeric shell conversions Parse shell text, identifiers, and related parser conversions with invariant numeric styles so decimal text is not interpreted through the current culture. Add de-DE regression coverage for text decimal arithmetic. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Parser/CultureInvariantConversionTests.cs | 98 +++++++++++++++++++ .../ExpressionParser.cs | 10 +- .../ShellObject/ShellIdentifier.cs | 5 +- .../ShellObject/ShellJson.cs | 5 +- .../ShellObject/ShellObject.cs | 2 +- .../ShellObject/ShellText.cs | 5 +- docs/programming.md | 2 +- 7 files changed, 115 insertions(+), 12 deletions(-) create mode 100644 CosmosDBShell.Tests/Parser/CultureInvariantConversionTests.cs diff --git a/CosmosDBShell.Tests/Parser/CultureInvariantConversionTests.cs b/CosmosDBShell.Tests/Parser/CultureInvariantConversionTests.cs new file mode 100644 index 00000000..1e7c05d8 --- /dev/null +++ b/CosmosDBShell.Tests/Parser/CultureInvariantConversionTests.cs @@ -0,0 +1,98 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace CosmosShell.Tests.Parser; + +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; + +using Azure.Data.Cosmos.Shell.Core; +using Azure.Data.Cosmos.Shell.Parser; + +[Collection(CosmosShell.Tests.Shell.ThemeStateTestCollection.Name)] +public class CultureInvariantConversionTests +{ + [Fact] + public void ShellText_DecimalConversion_UsesInvariantCulture() + { + var value = WithGermanCulture(() => + Assert.IsType(new ShellText("1.5").ConvertShellObject(DataType.Decimal))); + + Assert.Equal(1.5d, value); + } + + [Fact] + public void ShellText_DecimalConversion_RejectsLocalizedDecimalSeparator() + { + WithGermanCulture(() => + Assert.Throws(() => new ShellText("1,5").ConvertShellObject(DataType.Decimal))); + } + + [Fact] + public void ShellIdentifier_DecimalConversion_UsesInvariantCulture() + { + var value = WithGermanCulture(() => + Assert.IsType(new ShellIdentifier("1.5").ConvertShellObject(DataType.Decimal))); + + Assert.Equal(1.5d, value); + + WithGermanCulture(() => + Assert.Throws(() => new ShellIdentifier("1,5").ConvertShellObject(DataType.Decimal))); + } + + [Fact] + public async Task ExpressionEvaluation_TextDecimalArithmetic_UsesInvariantCulture() + { + var result = await WithGermanCultureAsync(() => EvaluateExpressionAsync("\"1.5\" * 2.0")); + var value = Assert.IsType(result); + + Assert.Equal(3.0d, value.Value); + } + + private static T WithGermanCulture(Func action) + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; + + try + { + var culture = CultureInfo.GetCultureInfo("de-DE"); + CultureInfo.CurrentCulture = culture; + CultureInfo.CurrentUICulture = culture; + return action(); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; + } + } + + private static async Task WithGermanCultureAsync(Func> action) + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; + + try + { + var culture = CultureInfo.GetCultureInfo("de-DE"); + CultureInfo.CurrentCulture = culture; + CultureInfo.CurrentUICulture = culture; + return await action(); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; + } + } + + private static async Task EvaluateExpressionAsync(string input) + { + var expression = new ExpressionParser(new Lexer(input)).ParseFilterExpression(); + using var shell = ShellInterpreter.CreateInstance(); + return await expression.EvaluateAsync(shell, new CommandState(), CancellationToken.None); + } +} diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs index 04e85bbf..09a0ffe4 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs @@ -4,6 +4,8 @@ namespace Azure.Data.Cosmos.Shell.Parser; +using System.Globalization; + using Azure.Data.Cosmos.Shell.ArgumentParser; using Azure.Data.Cosmos.Shell.Util; @@ -673,7 +675,7 @@ private Expression ParsePrimaryCore() this.Advance(); - if (int.TryParse(token.Value, out int intValue)) + if (int.TryParse(token.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int intValue)) { return new ConstantExpression(token, new ShellNumber(intValue)); } @@ -693,7 +695,7 @@ private Expression ParsePrimaryCore() this.Advance(); - if (double.TryParse(token.Value, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out double doubleValue)) + if (double.TryParse(token.Value, NumberStyles.Float, CultureInfo.InvariantCulture, out double doubleValue)) { return new ConstantExpression(token, new ShellDecimal(doubleValue)); } @@ -1641,7 +1643,7 @@ private Expression ParseFilterPathExpression(Token firstToken) } var indexToken = this.Consume(TokenType.Number, MessageService.GetString("expression_error_expected_array_index")); - int index = int.TryParse(indexToken.Value, out var parsedIndex) ? parsedIndex : 0; + int index = int.TryParse(indexToken.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedIndex) ? parsedIndex : 0; var indexedCloseBracket = this.Consume(TokenType.CloseBracket, MessageService.GetString("expression_error_expected_close_bracket")); var indexQuestionToken = this.TryConsumeQuestion(); end = indexedCloseBracket.Start + indexedCloseBracket.Length; @@ -1745,4 +1747,4 @@ public override void Visit(JsonExpression jsonExpression) } } } -} \ No newline at end of file +} diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellIdentifier.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellIdentifier.cs index c0e22556..1e294117 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellIdentifier.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellIdentifier.cs @@ -4,6 +4,7 @@ namespace Azure.Data.Cosmos.Shell.Parser; +using System.Globalization; using System.Text.Json; using Azure.Data.Cosmos.Shell.Core; @@ -49,14 +50,14 @@ internal static object EvaluateString(DataType type, string value) throw new InvalidOperationException(MessageService.GetArgsString("conversion-error-identifier-boolean", "value", value)); case DataType.Number: - if (int.TryParse(value, out int intValue)) + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int intValue)) { return intValue; } throw new InvalidOperationException(MessageService.GetArgsString("conversion-error-identifier-number", "value", value)); case DataType.Decimal: - if (double.TryParse(value, out double decimalValue)) + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double decimalValue)) { return decimalValue; } diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellJson.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellJson.cs index fcde1ad0..ca9dd00c 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellJson.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellJson.cs @@ -4,6 +4,7 @@ namespace Azure.Data.Cosmos.Shell.Parser; +using System.Globalization; using System.Text.Json; using Azure.Data.Cosmos.Shell.Core; @@ -80,7 +81,7 @@ public override object ConvertShellObject(DataType type) if (this.Value.ValueKind == JsonValueKind.String) { var strValue = this.Value.GetString() ?? string.Empty; - if (int.TryParse(strValue, out int intValue)) + if (int.TryParse(strValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out int intValue)) { return intValue; } @@ -96,7 +97,7 @@ public override object ConvertShellObject(DataType type) if (this.Value.ValueKind == JsonValueKind.String) { var strValue = this.Value.GetString() ?? string.Empty; - if (decimal.TryParse(strValue, out decimal decimalValue)) + if (decimal.TryParse(strValue, NumberStyles.Float, CultureInfo.InvariantCulture, out decimal decimalValue)) { return decimalValue; } diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellObject.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellObject.cs index 972bb019..5fc7dab6 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellObject.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellObject.cs @@ -44,7 +44,7 @@ public static ShellObject Parse(Lexer lexer) } // Check if it's a number - if (int.TryParse(token.Value, out int intValue)) + if (int.TryParse(token.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int intValue)) { return new ShellNumber(intValue); } diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellText.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellText.cs index 193fc3d4..3d0e161f 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellText.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellText.cs @@ -4,6 +4,7 @@ namespace Azure.Data.Cosmos.Shell.Parser; +using System.Globalization; using System.Text.Json; using Azure.Data.Cosmos.Shell.Core; @@ -58,14 +59,14 @@ internal static object EvaluateString(DataType type, string value) throw new InvalidOperationException(MessageService.GetArgsString("conversion-error-text-boolean", "value", value)); case DataType.Number: - if (int.TryParse(value, out int intValue)) + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int intValue)) { return intValue; } throw new InvalidOperationException(MessageService.GetArgsString("conversion-error-text-number", "value", value)); case DataType.Decimal: - if (double.TryParse(value, out double decimalValue)) + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double decimalValue)) { return decimalValue; } diff --git a/docs/programming.md b/docs/programming.md index 73f178cb..4ef48e4b 100644 --- a/docs/programming.md +++ b/docs/programming.md @@ -28,7 +28,7 @@ Variable names are case-sensitive. For compatibility, the lexer also accepts hyp - Negatives: `-1` - Decimal literals: `3.14`, `3.0` -Integer values use signed 32-bit storage. Integer literal magnitudes must be between `0` and `2147483647`; the minus sign is a separate unary operator, so `-2147483648` is not accepted as a literal. The minimum integer value can be computed as `-2147483647 - 1`. Arithmetic between integers stays integer arithmetic, including truncating division (`3 / 2` is `1`). Integer overflow raises an error instead of wrapping. Use a decimal operand for floating-point arithmetic (`3.0 / 2` is `1.5`). Decimal values use IEEE 754 `double`, not exact base-10 decimal arithmetic. +Integer values use signed 32-bit storage. Integer literal magnitudes must be between `0` and `2147483647`; the minus sign is a separate unary operator, so `-2147483648` is not accepted as a literal. The minimum integer value can be computed as `-2147483647 - 1`. Arithmetic between integers stays integer arithmetic, including truncating division (`3 / 2` is `1`). Integer overflow raises an error instead of wrapping. Use a decimal operand for floating-point arithmetic (`3.0 / 2` is `1.5`). Decimal values use IEEE 754 `double`, not exact base-10 decimal arithmetic. Numeric text conversion is culture-invariant and uses `.` as the decimal separator. JSON numbers use the same rules in expressions and `for` loops: integer-form values within the `Int32` range become integers; fractional, exponent-form, or larger values use `double`. Large JSON integers can therefore lose precision beyond the exact range of `double`. For example, a JSON property containing `3` divided by `2` produces `1`, while a property containing `3.0` produces `1.5`. From a93f20ce58be0015c8ca2c1aafac5432e7e00bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Kr=C3=BCger?= Date: Tue, 29 Sep 2026 14:57:26 +0200 Subject: [PATCH 2/2] Accept large JSON integer literals Preserve out-of-range integer-form JSON literal values as raw JSON numbers while keeping plain integer expressions limited to Int32. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CosmosDBShell.Tests/Parser/ExpressionTests.cs | 50 +++++++++++ .../ExpressionParser.cs | 83 ++++++++++++++++++- docs/programming.md | 2 + 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/CosmosDBShell.Tests/Parser/ExpressionTests.cs b/CosmosDBShell.Tests/Parser/ExpressionTests.cs index c95fcc66..f8e231d8 100644 --- a/CosmosDBShell.Tests/Parser/ExpressionTests.cs +++ b/CosmosDBShell.Tests/Parser/ExpressionTests.cs @@ -491,6 +491,56 @@ public void EvaluateExpression_JsonObject_ReturnsShellJson() Assert.Equal(30, json.Value.GetProperty("age").GetInt32()); } + [Fact] + public void EvaluateExpression_JsonObjectWithLargeIntegers_PreservesRawIntegerText() + { + var result = EvaluateExpression("{ \"ts\": 1727600000000, \"neg\": -1727600000000 }"); + var json = Assert.IsType(result); + Assert.Equal("1727600000000", json.Value.GetProperty("ts").GetRawText()); + Assert.Equal("-1727600000000", json.Value.GetProperty("neg").GetRawText()); + } + + [Fact] + public void EvaluateExpression_JsonArrayWithLargeIntegers_PreservesRawIntegerText() + { + var result = EvaluateExpression("[2147483648, -1727600000000, 9007199254740993]"); + var json = Assert.IsType(result); + Assert.Equal("2147483648", json.Value[0].GetRawText()); + Assert.Equal("-1727600000000", json.Value[1].GetRawText()); + Assert.Equal("9007199254740993", json.Value[2].GetRawText()); + } + + [Fact] + public void EvaluateExpression_NestedJsonObjectWithLargeInteger_PreservesRawIntegerText() + { + var result = EvaluateExpression("{ \"outer\": { \"n\": 9007199254740993 } }"); + var json = Assert.IsType(result); + Assert.Equal("9007199254740993", json.Value.GetProperty("outer").GetProperty("n").GetRawText()); + } + + [Fact] + public async Task EvaluateExpression_LargeJsonIntegerPropertyArithmetic_UsesDecimalArithmetic() + { + var objectResult = Assert.IsType(await EvaluateExpressionAsync("{ \"n\": 3000000000 }")); + var interpreter = new ShellInterpreter(); + interpreter.SetVariable("o", objectResult); + + var result = await ParseExpression("$o.n + 1").EvaluateAsync(interpreter, new CommandState(), CancellationToken.None); + var number = Assert.IsType(result); + Assert.Equal(3000000001.0, number.Value); + } + + [Fact] + public void ParseExpression_LargeIntegerOutsideJson_ReportsDocumentedError() + { + var lexer = new Lexer("2147483648"); + var parser = new ExpressionParser(lexer); + var expression = parser.ParseExpression(); + + Assert.IsType(expression); + Assert.Contains(lexer.Errors, error => error.Message == MessageService.GetArgsString("expression_error_invalid_number", "value", "2147483648")); + } + [Fact] public void EvaluateExpression_JsonArray_ReturnsShellJson() { diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs index 09a0ffe4..445100cb 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs @@ -5,6 +5,7 @@ namespace Azure.Data.Cosmos.Shell.Parser; using System.Globalization; +using System.Text.Json; using Azure.Data.Cosmos.Shell.ArgumentParser; using Azure.Data.Cosmos.Shell.Util; @@ -1235,7 +1236,7 @@ void SkipTrivia() } // Parse property value as a full expression - var value = this.ParsePipeExpression(); + var value = this.ParseJsonValueExpression(); // Add to properties if key valid JsonProperty? propertyNode = null; @@ -1364,7 +1365,7 @@ private Expression ParseJsonArray() } // Parse next element as a full expression - var expr = this.ParsePipeExpression(); + var expr = this.ParseJsonValueExpression(); elements.Add(expr); this.SkipWhitespace(); @@ -1432,6 +1433,84 @@ private Expression ParseJsonArray() return new JsonArrayExpression(lbracket, synthetic, elements, commaTokens); } + private Expression ParseJsonValueExpression() + { + if (this.TryParseLargeJsonIntegerLiteral(out var expression)) + { + return expression!; + } + + return this.ParsePipeExpression(); + } + + private bool TryParseLargeJsonIntegerLiteral(out Expression? expression) + { + expression = null; + if (this.aborted) + { + return false; + } + + if (this.Check(TokenType.Number)) + { + var token = this.Current; + if (token == null || int.TryParse(token.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out _)) + { + return false; + } + + expression = this.TryCreateJsonIntegerConstant(token, token.Value); + if (expression == null) + { + return false; + } + + this.Advance(); + return true; + } + + if (this.Check(TokenType.Minus)) + { + var minusToken = this.Current; + var numberToken = this.Peek(); + if (minusToken == null || + numberToken?.Type != TokenType.Number || + int.TryParse(numberToken.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out _)) + { + return false; + } + + var value = "-" + numberToken.Value; + var token = new Token(TokenType.Number, value, minusToken.Start, numberToken.End - minusToken.Start); + expression = this.TryCreateJsonIntegerConstant(token, value); + if (expression == null) + { + return false; + } + + this.Advance(); + this.Advance(); + return true; + } + + return false; + } + + private Expression? TryCreateJsonIntegerConstant(Token token, string value) + { + try + { + using var document = JsonDocument.Parse(value); + return document.RootElement.ValueKind == JsonValueKind.Number + ? new ConstantExpression(token, new ShellJson(document.RootElement.Clone())) + : null; + } + catch (JsonException) + { + return null; + } + } + private void ReportError(string message, Token? token, TokenType? expected = null, int? position = null, ParseErrorKind kind = ParseErrorKind.Generic) { int start = position ?? token?.Start ?? (this.lastNonNullToken?.Start ?? 0); diff --git a/docs/programming.md b/docs/programming.md index 4ef48e4b..f7d15ed5 100644 --- a/docs/programming.md +++ b/docs/programming.md @@ -34,6 +34,8 @@ JSON numbers use the same rules in expressions and `for` loops: integer-form val JSON construction preserves the distinction between integers and shell decimals. Integral decimal values retain a fractional suffix or exponent, so `$object = {"value":3.0}` stores JSON `3.0` and `$object.value / 2` produces `1.5`. This also applies to arrays, computed decimal values, and repeated reconstruction after loop or function calls. Integer values remain JSON integers. The original spelling and number of trailing zeros are not preserved; values still use IEEE 754 `double` precision. +JSON object and array literals accept larger integer-form values and preserve them as JSON integers. + Numeric Boolean conversion uses zero versus nonzero, including for fractional and large JSON numbers. JSON numbers use the same `double` conversion as decimal shell values for this check, so `if 1.5` and `if $object.value` behave alike when the property contains `1.5`. JSON `null` remains JSON `null` when bound by a `for` loop or passed through a function. Rebuilding an array from that value produces `[null]`, not `["null"]`. Text conversion remains explicit and separate from JSON type preservation.