From f23b5c802c25258c8948e30fa0e6949781476857 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Mon, 19 Oct 2026 15:15:12 -0400 Subject: [PATCH] fix(ipc): Write one zero offset for empty variable-size arrays The array view of an empty string, binary, list or map array has a zero-length offsets buffer, and the encoder wrote that buffer as is. IPC permits it, but arrow-go and Apache.Arrow for .NET read the first offset unconditionally. Arrow C++ writes one zero offset; the encoder now does too. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/nanoarrow/ipc/decoder_test.cc | 9 ++++ src/nanoarrow/ipc/encoder.c | 17 ++++++- src/nanoarrow/ipc/encoder_test.cc | 78 +++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 24acafe64..8f89acc6f 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -1592,6 +1592,15 @@ void AssertArrayViewIdentical(const struct ArrowArrayView* actual, for (int i = 0; i < 3; i++) { auto a_buf = actual->buffer_views[i]; auto e_buf = expected->buffer_views[i]; + // The encoder writes one zero offset for an empty array, whose view has a + // zero-length offsets buffer + if (expected->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && + expected->length == 0 && e_buf.size_bytes == 0) { + ASSERT_EQ(a_buf.size_bytes, expected->layout.element_size_bits[i] / 8); + const uint8_t zeros[8] = {0}; + ASSERT_EQ(memcmp(a_buf.data.data, zeros, a_buf.size_bytes), 0); + continue; + } ASSERT_EQ(a_buf.size_bytes, e_buf.size_bytes); if (a_buf.size_bytes != 0) { ASSERT_EQ(memcmp(a_buf.data.data, e_buf.data.data, a_buf.size_bytes), 0); diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index de325c19c..2cccd33d0 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -965,6 +965,9 @@ static ArrowErrorCode ArrowIpcEncoderEncodeBodyCompression( return NANOARROW_OK; } +// One zero offset, large enough for either 32- or 64-bit offsets +static const int64_t kZeroOffsets[1] = {0}; + // Collect the node and buffer views of an array to be encoded static ArrowErrorCode ArrowIpcEncoderCollectArray(struct ArrowIpcEncoderPrivate* private, const struct ArrowArrayView* array_view, @@ -982,8 +985,20 @@ static ArrowErrorCode ArrowIpcEncoderCollectArray(struct ArrowIpcEncoderPrivate* NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowBufferAppend(&private->nodes, &node, sizeof(node)), error); for (int64_t b = 0; b < array_view->array->n_buffers; ++b) { + struct ArrowBufferView buffer_view = array_view->buffer_views[b]; + + // An offsets buffer holds length + 1 offsets, so even an empty array has one. + // The array view of an empty array has a zero-length offsets buffer (IPC permits + // one), but some readers (arrow-go, Apache.Arrow for .NET) read the first offset + // unconditionally. Write a single zero offset instead, as Arrow C++ does. + if (array_view->layout.buffer_type[b] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && + array_view->length == 0 && buffer_view.size_bytes == 0) { + buffer_view.data.data = kZeroOffsets; + buffer_view.size_bytes = array_view->layout.element_size_bits[b] / 8; + } + NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowBufferAppend(&private->buffer_views, &array_view->buffer_views[b], + ArrowBufferAppend(&private->buffer_views, &buffer_view, sizeof(struct ArrowBufferView)), error); } diff --git a/src/nanoarrow/ipc/encoder_test.cc b/src/nanoarrow/ipc/encoder_test.cc index 54dd4547c..d17782966 100644 --- a/src/nanoarrow/ipc/encoder_test.cc +++ b/src/nanoarrow/ipc/encoder_test.cc @@ -853,6 +853,84 @@ TEST(NanoarrowIpcTest, NanoarrowIpcEncoderWritesActualNullCounts) { EXPECT_EQ(decoded->children[1]->null_count, 3); } +TEST(NanoarrowIpcTest, NanoarrowIpcEncoderWritesOffsetForEmptyArrays) { + // An empty record batch of struct>. The array view of an empty array has a zero-length offsets + // buffer, whatever the array itself holds. + struct ArrowError error; + nanoarrow::UniqueSchema schema; + ASSERT_EQ(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateChildren(schema.get(), 4), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_STRING), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[0], "s"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->children[1], NANOARROW_TYPE_BINARY), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[1], "b"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->children[2], NANOARROW_TYPE_LARGE_STRING), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[2], "ls"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->children[3], NANOARROW_TYPE_LIST), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[3], "l"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[3]->children[0], NANOARROW_TYPE_INT32), + NANOARROW_OK); + + nanoarrow::UniqueArray array; + ASSERT_EQ(ArrowArrayInitFromSchema(array.get(), schema.get(), &error), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishBuildingDefault(array.get(), &error), NANOARROW_OK) + << error.message; + + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), &error), NANOARROW_OK) + << error.message; + for (int64_t i = 0; i < array_view->n_children; i++) { + ASSERT_EQ(array_view->children[i]->buffer_views[1].size_bytes, 0); + } + + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + nanoarrow::UniqueBuffer message, body; + ASSERT_EQ(ArrowIpcEncoderEncodeSimpleRecordBatch(encoder.get(), array_view.get(), + body.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ( + ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, message.get()), + NANOARROW_OK); + + // The message carries one zero offset for each empty array + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), NANOARROW_OK) + << error.message; + struct ArrowBufferView message_view = {{message->data}, message->size_bytes}; + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), message_view, &error), + NANOARROW_OK) + << error.message; + struct ArrowArrayView* decoded; + struct ArrowBufferView body_view = {{body->data}, body->size_bytes}; + ASSERT_EQ( + ArrowIpcDecoderDecodeArrayView(decoder.get(), body_view, -1, &decoded, &error), + NANOARROW_OK) + << error.message; + + const int64_t offset_sizes[] = {4, 4, 8, 4}; + for (int64_t i = 0; i < decoded->n_children; i++) { + SCOPED_TRACE(schema->children[i]->name); + const struct ArrowBufferView& offsets = decoded->children[i]->buffer_views[1]; + ASSERT_EQ(offsets.size_bytes, offset_sizes[i]); + if (offset_sizes[i] == 8) { + EXPECT_EQ(offsets.data.as_int64[0], 0); + } else { + EXPECT_EQ(offsets.data.as_int32[0], 0); + } + } +} + TEST(NanoarrowIpcTest, NanoarrowIpcEncoderUncompressedRecordBatchAllocation) { nanoarrow::ipc::UniqueEncoder encoder; ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK);