From 0879cf2753e57cf2fb25e96b1c4afeb510b9c6c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Kr=C3=BCger?= Date: Tue, 29 Sep 2026 14:55:27 +0200 Subject: [PATCH 1/2] Handle scalar rows in CSV export Add a value column for non-object CSV export rows so scalar and mixed query results keep their data while preserving existing object row behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CommandTests/ExportCommandTests.cs | 82 +++++++++++++++++++ .../ExportCommand.cs | 20 ++++- docs/commands.md | 2 +- 3 files changed, 101 insertions(+), 3 deletions(-) diff --git a/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs b/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs index fe90cf7..b8f0ea6 100644 --- a/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs +++ b/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs @@ -204,6 +204,88 @@ public async Task WriteCsvAsync_NestedValuesWrittenAsCompactJson() Assert.Contains("\"{\"\"a\"\":\"\"b\"\"}\"", output); } + [Fact] + public async Task WriteCsvAsync_ObjectRowsKeepRawJsonValues() + { + var items = ToAsyncEnumerableAsync( + JsonSerializer.SerializeToElement(new { id = "1", flag = true, missing = (object?)null })); + + using var writer = new StringWriter(); + writer.NewLine = "\n"; + + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + + Assert.Equal(1, count); + var lines = writer.ToString().TrimEnd('\n').Split('\n'); + Assert.Equal("\"id\",\"flag\",\"missing\"", lines[0]); + Assert.Equal("\"1\",\"true\",\"null\"", lines[1]); + } + + [Fact] + public async Task WriteCsvAsync_ScalarStringsUseValueColumnAndEscapeCsv() + { + var items = ToAsyncEnumerableAsync( + JsonSerializer.SerializeToElement("a,\"b")); + + using var writer = new StringWriter(); + writer.NewLine = "\n"; + + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + + Assert.Equal(1, count); + Assert.Equal("\"value\"\n\"a,\"\"b\"\n", writer.ToString()); + } + + [Fact] + public async Task WriteCsvAsync_ScalarNumbersBoolsAndNullUseValueColumn() + { + var items = ToAsyncEnumerableAsync( + JsonSerializer.SerializeToElement(42), + JsonSerializer.SerializeToElement(true), + JsonSerializer.SerializeToElement(false), + JsonSerializer.SerializeToElement((object?)null)); + + using var writer = new StringWriter(); + writer.NewLine = "\n"; + + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + + Assert.Equal(4, count); + Assert.Equal("\"value\"\n\"42\"\n\"True\"\n\"False\"\n\"\"\n", writer.ToString()); + } + + [Fact] + public async Task WriteCsvAsync_ArrayRowsUseValueColumn() + { + var items = ToAsyncEnumerableAsync( + JsonSerializer.SerializeToElement(new[] { 1, 2 })); + + using var writer = new StringWriter(); + writer.NewLine = "\n"; + + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + + Assert.Equal(1, count); + Assert.Equal("\"value\"\n\"[1,2]\"\n", writer.ToString()); + } + + [Fact] + public async Task WriteCsvAsync_MixedObjectAndScalarRowsPreserveScalarValue() + { + var items = ToAsyncEnumerableAsync( + JsonSerializer.SerializeToElement(new { id = "1" }), + JsonSerializer.SerializeToElement("a,b"), + JsonSerializer.SerializeToElement(new { id = "2", value = 99 })); + + using var writer = new StringWriter(); + writer.NewLine = "\n"; + + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + + Assert.Equal(3, count); + Assert.Equal("\"id\",\"value\"\n\"1\",\"\"\n\"\",\"a,b\"\n\"2\",\"99\"\n", writer.ToString()); + } + [Fact] public async Task WriteCsvAsync_WithNoItems_ProducesEmptyOutput() { diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs index 1d1707d..126c091 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs @@ -37,6 +37,7 @@ internal enum ExportFormat internal class ExportCommand : CosmosCommand { private const string DefaultQuery = "SELECT * FROM c"; + private const string ValueColumnName = "value"; [CosmosParameter("file", RequiredErrorKey = "command-export-error-missing_file")] public string? File { get; init; } @@ -187,8 +188,9 @@ internal static async Task WriteArrayAsync(IAsyncEnumerable it /// /// Writes a sequence of items to as CSV. The header row is the /// union of all top-level property names (in first-seen order); each subsequent row - /// contains the corresponding values. Nested objects and arrays are written as compact - /// JSON. Items are spooled to disk to compute the column set. + /// contains the corresponding values. Non-object rows are written in a value + /// column. Nested objects and arrays are written as compact JSON. Items are spooled to + /// disk to compute the column set. /// /// The items to write. /// The destination writer. @@ -218,6 +220,11 @@ internal static async Task WriteCsvAsync(IAsyncEnumerable item await spoolWriter.WriteLineAsync(SerializeJsonLine(item).AsMemory(), token); if (item.ValueKind != JsonValueKind.Object) { + if (headerSet.Add(ValueColumnName)) + { + headers.Add(ValueColumnName); + } + continue; } @@ -267,6 +274,10 @@ internal static async Task WriteCsvAsync(IAsyncEnumerable item var text = value.ValueKind == JsonValueKind.String ? value.GetString() ?? string.Empty : value.GetRawText(); sb.Append(CommandState.EscapeCSV(text)); } + else if (item.ValueKind != JsonValueKind.Object && headers[i] == ValueColumnName) + { + sb.Append(CommandState.EscapeCSV(GetScalarCsvValue(item))); + } else { sb.Append(CommandState.EscapeCSV(string.Empty)); @@ -281,6 +292,11 @@ internal static async Task WriteCsvAsync(IAsyncEnumerable item return count; } + private static string GetScalarCsvValue(JsonElement item) + { + return item.ToString(); + } + private static async Task<(int Count, double Charge)> ExecuteExportAsync( Container container, string query, diff --git a/docs/commands.md b/docs/commands.md index dd6a476..8e63e9c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -654,7 +654,7 @@ Examples: ### export -Stream items from a container to a local file. Default format is JSON Lines (one compact JSON object per line); pass `--format=array` for a single JSON array, or `--format=csv` for CSV. JSON formats stream incrementally. CSV spools documents to a private temporary file to compute the complete column set, keeping only the column names and current record in memory. Allow enough temporary disk space for the JSON spool as well as the destination export. The CSV separator follows the `COSMOSDB_SHELL_CSVSEP` environment variable (default `;`). +Stream items from a container to a local file. Default format is JSON Lines (one compact JSON object per line); pass `--format=array` for a single JSON array, or `--format=csv` for CSV. JSON formats stream incrementally. CSV spools documents to a private temporary file to compute the complete column set, keeping only the column names and current record in memory. Allow enough temporary disk space for the JSON spool as well as the destination export. The CSV separator follows the `COSMOSDB_SHELL_CSVSEP` environment variable (default `;`). Non-object query results are written to a CSV `value` column; mixed object and non-object results include both object property columns and the `value` column. All formats write to a temporary file in the destination directory and move it into place only after successful completion. An existing destination requires `--force` and is preserved if reading, writing, or cancellation interrupts the export. Temporary files are removed on normal completion and handled failures; an abrupt process termination can leave an unfinished destination-directory temporary file. Once `--max` items have been emitted, no further query pages are requested. From a03a62c8c7d6095741d2a4b04474ef9bfb7e4883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Kr=C3=BCger?= Date: Wed, 30 Sep 2026 14:49:07 +0200 Subject: [PATCH 2/2] Use configurable scalar CSV header and reject unnamed import values Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CosmosDBShell.Fuzzer/ImportFuzzer.cs | 2 +- .../CommandTests/ExportCommandTests.cs | 54 ++++++++++++--- .../CommandTests/ImportCommandTests.cs | 67 +++++++++++++++++-- .../ExportCommand.cs | 29 +++++--- .../ImportCommand.cs | 17 ++++- CosmosDBShell/lang/en.ftl | 2 + README.md | 2 +- docs/commands.md | 4 +- docs/navigation.md | 1 + l10n/CosmosDBShell.json | 2 + 10 files changed, 150 insertions(+), 30 deletions(-) diff --git a/CosmosDBShell.Fuzzer/ImportFuzzer.cs b/CosmosDBShell.Fuzzer/ImportFuzzer.cs index 4a7ac15..edf80ba 100644 --- a/CosmosDBShell.Fuzzer/ImportFuzzer.cs +++ b/CosmosDBShell.Fuzzer/ImportFuzzer.cs @@ -116,7 +116,7 @@ private static async Task DriveParsersAsync(string content) var pk = ImportCommand.ParsePartitionKeySegments(RandomPartitionKey()); for (var r = 1; r < records.Count; r++) { - _ = ImportCommand.BuildCsvObject(headers, records[r], pk); + _ = ImportCommand.BuildCsvObject(headers, records[r], pk, r + 1); } } } diff --git a/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs b/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs index b8f0ea6..3d2fa9d 100644 --- a/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs +++ b/CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs @@ -10,6 +10,7 @@ namespace CosmosShell.Tests.CommandTests; using System.Threading; using System.Threading.Tasks; using Azure.Data.Cosmos.Shell.Commands; +using Azure.Data.Cosmos.Shell.Core; using Azure.Data.Cosmos.Shell.Util; using Microsoft.Azure.Cosmos; using NSubstitute; @@ -222,7 +223,7 @@ public async Task WriteCsvAsync_ObjectRowsKeepRawJsonValues() } [Fact] - public async Task WriteCsvAsync_ScalarStringsUseValueColumnAndEscapeCsv() + public async Task WriteCsvAsync_ScalarStringsUseEmptyHeaderAndEscapeCsv() { var items = ToAsyncEnumerableAsync( JsonSerializer.SerializeToElement("a,\"b")); @@ -230,14 +231,14 @@ public async Task WriteCsvAsync_ScalarStringsUseValueColumnAndEscapeCsv() using var writer = new StringWriter(); writer.NewLine = "\n"; - var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty); Assert.Equal(1, count); - Assert.Equal("\"value\"\n\"a,\"\"b\"\n", writer.ToString()); + Assert.Equal("\"\"\n\"a,\"\"b\"\n", writer.ToString()); } [Fact] - public async Task WriteCsvAsync_ScalarNumbersBoolsAndNullUseValueColumn() + public async Task WriteCsvAsync_ScalarNumbersBoolsAndNullUseEmptyHeader() { var items = ToAsyncEnumerableAsync( JsonSerializer.SerializeToElement(42), @@ -248,14 +249,14 @@ public async Task WriteCsvAsync_ScalarNumbersBoolsAndNullUseValueColumn() using var writer = new StringWriter(); writer.NewLine = "\n"; - var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty); Assert.Equal(4, count); - Assert.Equal("\"value\"\n\"42\"\n\"True\"\n\"False\"\n\"\"\n", writer.ToString()); + Assert.Equal("\"\"\n\"42\"\n\"True\"\n\"False\"\n\"\"\n", writer.ToString()); } [Fact] - public async Task WriteCsvAsync_ArrayRowsUseValueColumn() + public async Task WriteCsvAsync_ArrayRowsUseEmptyHeader() { var items = ToAsyncEnumerableAsync( JsonSerializer.SerializeToElement(new[] { 1, 2 })); @@ -263,10 +264,10 @@ public async Task WriteCsvAsync_ArrayRowsUseValueColumn() using var writer = new StringWriter(); writer.NewLine = "\n"; - var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty); Assert.Equal(1, count); - Assert.Equal("\"value\"\n\"[1,2]\"\n", writer.ToString()); + Assert.Equal("\"\"\n\"[1,2]\"\n", writer.ToString()); } [Fact] @@ -280,10 +281,41 @@ public async Task WriteCsvAsync_MixedObjectAndScalarRowsPreserveScalarValue() using var writer = new StringWriter(); writer.NewLine = "\n"; - var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None); + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty); Assert.Equal(3, count); - Assert.Equal("\"id\",\"value\"\n\"1\",\"\"\n\"\",\"a,b\"\n\"2\",\"99\"\n", writer.ToString()); + Assert.Equal("\"id\",\"\",\"value\"\n\"1\",\"\",\"\"\n\"\",\"a,b\",\"\"\n\"2\",\"\",\"99\"\n", writer.ToString()); + } + + [Fact] + public async Task WriteCsvAsync_CustomScalarHeaderPreservesObjectValueColumn() + { + var items = ToAsyncEnumerableAsync( + JsonSerializer.SerializeToElement(new { value = 99 }), + JsonSerializer.SerializeToElement("text")); + using var writer = new StringWriter(); + writer.NewLine = "\n"; + + var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, "scalar"); + + Assert.Equal(2, count); + Assert.Equal("\"value\",\"scalar\"\n\"99\",\"\"\n\"\",\"text\"\n", writer.ToString()); + } + + [Theory] + [InlineData("", "{\"\":1}")] + [InlineData("value", "{\"value\":1}")] + public async Task WriteCsvAsync_ScalarHeaderConflictingWithObjectPropertyFails(string header, string objectJson) + { + using var document = JsonDocument.Parse(objectJson); + var items = ToAsyncEnumerableAsync(JsonSerializer.SerializeToElement(42), document.RootElement.Clone()); + using var writer = new StringWriter(); + + var error = await Assert.ThrowsAsync(() => + ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, header)); + + Assert.Contains("COSMOSDB_SHELL_CSV_SCALAR_COLUMN", error.Message); + Assert.Equal(string.Empty, writer.ToString()); } [Fact] diff --git a/CosmosDBShell.Tests/CommandTests/ImportCommandTests.cs b/CosmosDBShell.Tests/CommandTests/ImportCommandTests.cs index 6fbccca..65126c0 100644 --- a/CosmosDBShell.Tests/CommandTests/ImportCommandTests.cs +++ b/CosmosDBShell.Tests/CommandTests/ImportCommandTests.cs @@ -340,13 +340,66 @@ public void BuildCsvObject_MapsColumnsToStringProperties() var element = ImportCommand.BuildCsvObject( new[] { "id", "name" }, new[] { "1", "Alice" }, - partitionKeySegments: null); + partitionKeySegments: null, + lineNumber: 2); Assert.Equal(JsonValueKind.Object, element.ValueKind); Assert.Equal("1", element.GetProperty("id").GetString()); Assert.Equal("Alice", element.GetProperty("name").GetString()); } + [Fact] + public void BuildCsvObject_EmptyUnnamedColumnIsIgnored() + { + var element = ImportCommand.BuildCsvObject( + new[] { "id", "" }, + new[] { "1", "" }, + partitionKeySegments: null, + lineNumber: 2); + + Assert.Equal("1", element.GetProperty("id").GetString()); + Assert.False(element.TryGetProperty("", out _)); + } + + [Fact] + public void BuildCsvObject_PopulatedUnnamedColumnFailsWithLocation() + { + var error = Assert.Throws(() => ImportCommand.BuildCsvObject( + new[] { "id", "" }, + new[] { "1", "lost" }, + partitionKeySegments: null, + lineNumber: 7)); + + Assert.Contains("7", error.Message); + Assert.Contains("2", error.Message); + Assert.Contains("header", error.Message); + } + + [Fact] + public async Task EnumerateCsvAsync_PopulatedUnnamedColumnReportsPhysicalStartLine() + { + var separator = Azure.Data.Cosmos.Shell.Core.ShellInterpreter.CSVSeparator; + var filePath = Path.GetTempFileName(); + try + { + await File.WriteAllTextAsync(filePath, $"id{separator}\"\"\n\"a\nb\"{separator}\"\"\n2{separator}lost\n", TestContext.Current.CancellationToken); + + var error = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in ImportCommand.EnumerateCsvAsync(filePath, null, TestContext.Current.CancellationToken)) + { + } + }); + + Assert.Contains("4", error.Message); + Assert.Contains("2", error.Message); + } + finally + { + File.Delete(filePath); + } + } + [Fact] public void ReadCsvRecords_ReadsValidRowsBeforeReportingMalformedRecord() { @@ -464,7 +517,8 @@ public void BuildCsvObject_SingleSegmentPartitionKey_StaysTopLevel() var element = ImportCommand.BuildCsvObject( new[] { "id", "city" }, new[] { "1", "Seattle" }, - new[] { "city" }); + new[] { "city" }, + lineNumber: 2); Assert.Equal("Seattle", element.GetProperty("city").GetString()); } @@ -475,7 +529,8 @@ public void BuildCsvObject_NestedPartitionKey_NestsMatchingColumn() var element = ImportCommand.BuildCsvObject( new[] { "id", "city" }, new[] { "1", "Seattle" }, - new[] { "address", "city" }); + new[] { "address", "city" }, + lineNumber: 2); Assert.False(element.TryGetProperty("city", out _)); Assert.Equal("Seattle", element.GetProperty("address").GetProperty("city").GetString()); @@ -488,7 +543,8 @@ public void BuildCsvObject_NestedPartitionKey_ConflictingScalarColumn_Throws() var ex = Assert.Throws(() => ImportCommand.BuildCsvObject( new[] { "id", "address", "city" }, new[] { "1", "123 Main St", "Seattle" }, - new[] { "address", "city" })); + new[] { "address", "city" }, + lineNumber: 2)); Assert.Contains("address", ex.Message, StringComparison.Ordinal); } @@ -499,7 +555,8 @@ public void BuildCsvObject_MissingValues_FillWithEmptyString() var element = ImportCommand.BuildCsvObject( new[] { "id", "name", "extra" }, new[] { "1" }, - partitionKeySegments: null); + partitionKeySegments: null, + lineNumber: 2); Assert.Equal("1", element.GetProperty("id").GetString()); Assert.Equal(string.Empty, element.GetProperty("name").GetString()); diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs index 126c091..a2195c4 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ExportCommand.cs @@ -37,7 +37,7 @@ internal enum ExportFormat internal class ExportCommand : CosmosCommand { private const string DefaultQuery = "SELECT * FROM c"; - private const string ValueColumnName = "value"; + private const string ScalarColumnEnvironmentVariable = "COSMOSDB_SHELL_CSV_SCALAR_COLUMN"; [CosmosParameter("file", RequiredErrorKey = "command-export-error-missing_file")] public string? File { get; init; } @@ -188,17 +188,19 @@ internal static async Task WriteArrayAsync(IAsyncEnumerable it /// /// Writes a sequence of items to as CSV. The header row is the /// union of all top-level property names (in first-seen order); each subsequent row - /// contains the corresponding values. Non-object rows are written in a value - /// column. Nested objects and arrays are written as compact JSON. Items are spooled to - /// disk to compute the column set. + /// contains the corresponding values. Non-object rows are written in a configurable + /// column with an empty header by default. Nested objects and arrays are written as + /// compact JSON. Items are spooled to disk to compute the column set. /// /// The items to write. /// The destination writer. /// The field separator. /// Cancellation token. + /// The header for non-object rows; defaults to the environment setting. /// The number of data rows written. - internal static async Task WriteCsvAsync(IAsyncEnumerable items, TextWriter writer, char separator, CancellationToken token) + internal static async Task WriteCsvAsync(IAsyncEnumerable items, TextWriter writer, char separator, CancellationToken token, string? scalarColumnName = null) { + scalarColumnName ??= Environment.GetEnvironmentVariable(ScalarColumnEnvironmentVariable) ?? string.Empty; var spoolOptions = new FileStreamOptions { Mode = FileMode.CreateNew, @@ -215,14 +217,17 @@ internal static async Task WriteCsvAsync(IAsyncEnumerable item using var spoolWriter = new StreamWriter(spool, new UTF8Encoding(false), leaveOpen: true); var headers = new List(); var headerSet = new HashSet(StringComparer.Ordinal); + var objectHeaders = new HashSet(StringComparer.Ordinal); + var hasScalarRows = false; await foreach (var item in items.WithCancellation(token)) { await spoolWriter.WriteLineAsync(SerializeJsonLine(item).AsMemory(), token); if (item.ValueKind != JsonValueKind.Object) { - if (headerSet.Add(ValueColumnName)) + hasScalarRows = true; + if (headerSet.Add(scalarColumnName)) { - headers.Add(ValueColumnName); + headers.Add(scalarColumnName); } continue; @@ -230,6 +235,7 @@ internal static async Task WriteCsvAsync(IAsyncEnumerable item foreach (var prop in item.EnumerateObject()) { + objectHeaders.Add(prop.Name); if (headerSet.Add(prop.Name)) { headers.Add(prop.Name); @@ -237,6 +243,13 @@ internal static async Task WriteCsvAsync(IAsyncEnumerable item } } + if (hasScalarRows && objectHeaders.Contains(scalarColumnName)) + { + throw new CommandException( + "export", + MessageService.GetArgsString("command-export-error-scalar_column_conflict", "column", scalarColumnName)); + } + await spoolWriter.FlushAsync(token); spool.Position = 0; using var spoolReader = new StreamReader(spool, leaveOpen: true); @@ -274,7 +287,7 @@ internal static async Task WriteCsvAsync(IAsyncEnumerable item var text = value.ValueKind == JsonValueKind.String ? value.GetString() ?? string.Empty : value.GetRawText(); sb.Append(CommandState.EscapeCSV(text)); } - else if (item.ValueKind != JsonValueKind.Object && headers[i] == ValueColumnName) + else if (item.ValueKind != JsonValueKind.Object && headers[i] == scalarColumnName) { sb.Append(CommandState.EscapeCSV(GetScalarCsvValue(item))); } diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ImportCommand.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ImportCommand.cs index 466b7e5..0bf1a24 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ImportCommand.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/ImportCommand.cs @@ -341,8 +341,9 @@ private static CsvParser CreateCsvParser(TextReader reader, char separator, Canc /// The header field names. /// The row values. /// Optional partition key path segments. + /// The physical start line of the CSV record. /// The constructed JSON object element. - internal static JsonElement BuildCsvObject(IReadOnlyList headers, IReadOnlyList values, string[]? partitionKeySegments) + internal static JsonElement BuildCsvObject(IReadOnlyList headers, IReadOnlyList values, string[]? partitionKeySegments, int lineNumber) { var root = new JsonObject(); for (var i = 0; i < headers.Count; i++) @@ -350,6 +351,18 @@ internal static JsonElement BuildCsvObject(IReadOnlyList headers, IReadO var name = headers[i]; if (string.IsNullOrEmpty(name)) { + if (i < values.Count && !string.IsNullOrEmpty(values[i])) + { + throw new CommandException( + "import", + MessageService.GetArgsString( + "command-import-error-unnamed_csv_value", + "line", + lineNumber, + "column", + i + 1)); + } + continue; } @@ -426,7 +439,7 @@ internal static JsonElement BuildCsvObject(IReadOnlyList headers, IReadO continue; } - yield return (startLine, BuildCsvObject(headers, fields, partitionKeySegments)); + yield return (startLine, BuildCsvObject(headers, fields, partitionKeySegments, startLine)); } } diff --git a/CosmosDBShell/lang/en.ftl b/CosmosDBShell/lang/en.ftl index 75b83ed..38a766d 100644 --- a/CosmosDBShell/lang/en.ftl +++ b/CosmosDBShell/lang/en.ftl @@ -454,6 +454,7 @@ command-export-error-missing_file = A destination file path is required. command-export-error-file_exists = File '{ $file }' already exists. Use --force to overwrite. command-export-error-destination_directory = Destination '{ $file }' is a directory. Specify a file path instead. command-export-error-query_failed = Export query failed: { $status } - { $message } +command-export-error-scalar_column_conflict = CSV scalar column '{ $column }' conflicts with an object property. Set COSMOSDB_SHELL_CSV_SCALAR_COLUMN to a different name. command-import-description = Imports items into a container from a JSON Lines, JSON array, or CSV file. command-import-description-file = Source file path. @@ -479,6 +480,7 @@ command-import-dry-run-success = Dry run: { $count } valid { $count -> } command-import-error-missing_file = A source file path is required. command-import-error-invalid_csv = Invalid CSV record at line { $line }. +command-import-error-unnamed_csv_value = Line { $line }, column { $column } has a value but no CSV header. Add a column name before importing. script-error-argument-count = Function '{ $name }' expects { $expected } { $expected -> [one] argument *[other] arguments diff --git a/README.md b/README.md index 719a134..2e2aa77 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ A terminal-native shell for Azure Cosmos DB — navigate databases like a filesy - MCP server for AI/tool integration - Distributed tracing via OpenTelemetry (`--otel`): emits a sampled W3C `traceparent` on Cosmos requests, with optional OTLP export -Exports replace their destination only after successful completion, preserving an existing file on failure or cancellation. Imports stream records; CSV exports use temporary disk storage to discover columns without retaining all documents in memory. See [import/export](docs/commands.md#export). +Exports replace their destination only after successful completion, preserving an existing file on failure or cancellation. Imports stream records; CSV exports use temporary disk storage to discover columns without retaining all documents in memory. Scalar CSV results use an empty column header by default; set `COSMOSDB_SHELL_CSV_SCALAR_COLUMN` to name it. An import rejects populated columns with empty headers rather than discarding their values. See [import/export](docs/commands.md#export). MCP command execution is serialized with the shell, and destructive confirmations are invalidated by connection or navigation changes. MCP invocations are echoed in the shell so their activity stays visible, and they are recorded in history alongside interactive commands. History remains fully replayable, including connection strings; treat its file as sensitive. See [MCP security](docs/mcp.md#security) and [history](docs/navigation.md#history). diff --git a/docs/commands.md b/docs/commands.md index 8e63e9c..b42441f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -654,7 +654,7 @@ Examples: ### export -Stream items from a container to a local file. Default format is JSON Lines (one compact JSON object per line); pass `--format=array` for a single JSON array, or `--format=csv` for CSV. JSON formats stream incrementally. CSV spools documents to a private temporary file to compute the complete column set, keeping only the column names and current record in memory. Allow enough temporary disk space for the JSON spool as well as the destination export. The CSV separator follows the `COSMOSDB_SHELL_CSVSEP` environment variable (default `;`). Non-object query results are written to a CSV `value` column; mixed object and non-object results include both object property columns and the `value` column. +Stream items from a container to a local file. Default format is JSON Lines (one compact JSON object per line); pass `--format=array` for a single JSON array, or `--format=csv` for CSV. JSON formats stream incrementally. CSV spools documents to a private temporary file to compute the complete column set, keeping only the column names and current record in memory. Allow enough temporary disk space for the JSON spool as well as the destination export. The CSV separator follows the `COSMOSDB_SHELL_CSVSEP` environment variable (default `;`). Non-object query results are written to a column with an empty header by default; set `COSMOSDB_SHELL_CSV_SCALAR_COLUMN` to name it. Mixed object and non-object results include both object property columns and the scalar column. If its header conflicts with an object property, export fails instead of combining their values. Scalar CSV exports cannot be imported as items without supplying item IDs and any required partition keys. All formats write to a temporary file in the destination directory and move it into place only after successful completion. An existing destination requires `--force` and is preserved if reading, writing, or cancellation interrupts the export. Temporary files are removed on normal completion and handled failures; an abrupt process termination can leave an unfinished destination-directory temporary file. Once `--max` items have been emitted, no further query pages are requested. @@ -684,7 +684,7 @@ The summary line reports the number of items written and the total RU charge. ### import -Bulk-load items from a JSON Lines, JSON array, or CSV file into a container. Format is auto-detected: a `.csv` extension selects CSV, otherwise the first non-whitespace character is inspected (`[` ⇒ array, otherwise JSON Lines). It can be forced with `--format`. Default mode is `insert`; pass `--mode=upsert` to replace items that already exist. For CSV, the header row defines property names and every value is imported as a string; the CSV separator follows `COSMOSDB_SHELL_CSVSEP` (default `;`). All formats are read incrementally rather than loading the complete file into memory. CSV supports quoted separators, escaped quotes, and multiline fields; malformed records abort the import with their physical start line. Earlier writes are not rolled back, so use `--dry-run` first when the entire file must be validated before any writes. +Bulk-load items from a JSON Lines, JSON array, or CSV file into a container. Format is auto-detected: a `.csv` extension selects CSV, otherwise the first non-whitespace character is inspected (`[` ⇒ array, otherwise JSON Lines). It can be forced with `--format`. Default mode is `insert`; pass `--mode=upsert` to replace items that already exist. For CSV, the header row defines property names and every value is imported as a string; the CSV separator follows `COSMOSDB_SHELL_CSVSEP` (default `;`). Empty headers with empty cells are ignored; a value under an empty header fails import with its line and column instead of being silently discarded. All formats are read incrementally rather than loading the complete file into memory. CSV supports quoted separators, escaped quotes, and multiline fields; malformed records abort the import with their physical start line. Earlier writes are not rolled back, so use `--dry-run` first when the entire file must be validated before any writes. ```text Usage: import [options] diff --git a/docs/navigation.md b/docs/navigation.md index 2af408a..729f490 100644 --- a/docs/navigation.md +++ b/docs/navigation.md @@ -327,6 +327,7 @@ These values are a public contract. See the [CI/CD guide](ci.md#exit-code-contra | `COSMOSDB_SHELL_TOKEN` | Pre-obtained Entra ID access token (JWT) for single-shot auth | | `COSMOSDB_SHELL_ACCOUNT_KEY` | Account key for authentication | | `COSMOSDB_SHELL_CSVSEP` | CSV column separator | +| `COSMOSDB_SHELL_CSV_SCALAR_COLUMN` | Header for non-object CSV export rows (empty by default) | | `COSMOSDB_SHELL_FORMAT` | Default output format (`user`, `json`, `table`, `csv`) used when `--output` is not supplied. Supplies a format only — it does not enable machine mode | | `OTEL_EXPORTER_OTLP_ENDPOINT` | Default OTLP endpoint used by `--otel` when no endpoint is supplied | diff --git a/l10n/CosmosDBShell.json b/l10n/CosmosDBShell.json index e60f4e9..cf66d00 100644 --- a/l10n/CosmosDBShell.json +++ b/l10n/CosmosDBShell.json @@ -350,6 +350,7 @@ "command-export-error-file_exists": "File \u0027{0}\u0027 already exists. Use --force to overwrite.", "command-export-error-missing_file": "A destination file path is required.", "command-export-error-query_failed": "Export query failed: {0} - {1}", + "command-export-error-scalar_column_conflict": "CSV scalar column \u0027{0}\u0027 conflicts with an object property. Set COSMOSDB_SHELL_CSV_SCALAR_COLUMN to a different name.", "command-export-example-1": "Export every item in the current container as JSON Lines", "command-export-example-2": "Export the results of a query", "command-export-example-3": "Export as a JSON array, overwriting an existing file", @@ -423,6 +424,7 @@ "command-import-error-missing_file": "A source file path is required.", "command-import-error-not_object": "Line {0} is not a JSON object.", "command-import-error-some_failed": "Failed to import {0} of {1} items.", + "command-import-error-unnamed_csv_value": "Line {0}, column {1} has a value but no CSV header. Add a column name before importing.", "command-import-example-1": "Import items from a JSON Lines file (one JSON object per line)", "command-import-example-2": "Import items from a JSON array file", "command-import-example-3": "Import items from a CSV file (the header row defines property names)",