From a8cc12586d003db16b7b991aae3aaf98dfff99b4 Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Mon, 25 May 2026 10:34:40 -0500 Subject: [PATCH 1/3] Rewrite SymbolMappingMsg hd.length to match encoded layout The Databento Live gateway emits SymbolMappingMsg in v1 wire layout (hd.length=20, 80 bytes total) even when the consumer is writing into a v3 file. The encoder dispatched on metadata.version to pick the body layout (v1: 64-byte body; v2+: 160-byte body) but wrote the record's original hd.length unchanged. The on-disk header then disagreed with the body that followed it, mis-aligning the decoder and corrupting every subsequent record in the stream. Re-derive hd.length from the layout actually being serialized (20 for v1, 44 for v2+) before writing the record header. The struct itself carries only parsed values, so the input version doesn't matter; the output is always self-consistent with the file's metadata.version. Added a regression test that constructs a v1-wire SymbolMappingMsg (hd.length=20), writes it into a v3 file alongside a trailing TradeMsg, and verifies both records round-trip cleanly with the symbology record's length rewritten to 44 on disk. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/encode.jl | 18 ++++++++++++---- test/test_phase8.jl | 50 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/encode.jl b/src/encode.jl index f7f3095..07d22c7 100644 --- a/src/encode.jl +++ b/src/encode.jl @@ -791,11 +791,21 @@ function write_record_complex(encoder::DBNEncoder, record) elseif isa(record, SymbolMappingMsg) # Spec-compliant SymbolMappingMsg layout (depends on DBN version): - # v1: stype_in_symbol[22] | stype_out_symbol[22] | pad(4) | start_ts(8) | end_ts(8) - # (no explicit stype_in/stype_out bytes in v1) + # v1: stype_in_symbol[22] | stype_out_symbol[22] | pad(4) | start_ts(8) | end_ts(8) (body 64, total 80, hd.length 20) # v2+: stype_in(1) | stype_in_symbol[71] | stype_out(1) | stype_out_symbol[71] | - # start_ts(8) | end_ts(8) - write_record_header(io, record.hd) + # start_ts(8) | end_ts(8) (body 160, total 176, hd.length 44) + # + # The record's hd.length may reflect a different on-wire version than + # the file we're writing into — the Databento Live gateway emits v1 + # layout (hd.length = 20) even when the consumer is writing a v3 file. + # Re-derive the length from the layout we are about to write so the + # resulting record header matches the bytes that follow it. + body_bytes = encoder.metadata.version == 1 ? 64 : 160 + fixed_length = UInt8((16 + body_bytes) ÷ LENGTH_MULTIPLIER) + hd = record.hd + out_hd = hd.length == fixed_length ? hd : + RecordHeader(fixed_length, hd.rtype, hd.publisher_id, hd.instrument_id, hd.ts_event) + write_record_header(io, out_hd) if encoder.metadata.version == 1 _write_fixed_string(io, record.stype_in_symbol, 22) _write_fixed_string(io, record.stype_out_symbol, 22) diff --git a/test/test_phase8.jl b/test/test_phase8.jl index c492fef..1a66e9c 100644 --- a/test/test_phase8.jl +++ b/test/test_phase8.jl @@ -247,10 +247,56 @@ using Dates @test records[1].stype_in_symbol == "AAPL.NASDAQ" @test records[1].stype_out == SType.INSTRUMENT_ID @test records[1].stype_out_symbol == "12345" - + rm(test_file, force=true) end - + + @testset "SymbolMappingMsg cross-version (v1 wire → v3 file)" begin + # Live Databento gateway emits SymbolMappingMsg in v1 wire layout + # (hd.length=20, 80 bytes total) even when the consumer writes a + # v3 file. The encoder must re-derive the header length from the + # v2+ layout it is actually about to serialize (hd.length=44, + # 176 bytes total) so that the resulting record header matches + # the body and the file remains parseable. + v1_wire_length = UInt8(20) + mapping = SymbolMappingMsg( + RecordHeader(v1_wire_length, RType.SYMBOL_MAPPING_MSG, + UInt16(0), UInt32(1191182337), 1779203015494476543), + SType.INSTRUMENT_ID, + "SPX.OPT", + SType.INSTRUMENT_ID, + "SPX 271217C02800000", + Int64(-1), + Int64(-1), + ) + + # Pair with a follow-up record so we exercise the stream offset: + # if hd.length on the SymbolMappingMsg is wrong, the decoder + # mis-positions for the next read and the file is unparseable. + # TradeMsg on-wire: 16-byte header + 32-byte body = 48 bytes, hd.length = 12 + trade = TradeMsg( + RecordHeader(UInt8(12), RType.MBP_0_MSG, UInt16(0), UInt32(1191182337), + 1779203015494476600), + Int64(100_000_000_000), UInt32(10), + Action.TRADE, Side.BID, 0x00, 0x00, + Int64(1779203015494476600), Int32(0), UInt32(1), + ) + + write_dbn(test_file, metadata, [mapping, trade]) + records = read_dbn(test_file) + + @test length(records) == 2 + @test records[1] isa SymbolMappingMsg + @test records[1].stype_in_symbol == "SPX.OPT" + @test records[1].stype_out_symbol == "SPX 271217C02800000" + @test records[1].hd.instrument_id == UInt32(1191182337) + @test records[1].hd.length == UInt8(44) # rewritten to v3 layout + @test records[2] isa TradeMsg + @test records[2].hd.instrument_id == UInt32(1191182337) + + rm(test_file, force=true) + end + @testset "SystemMsg Write/Read" begin # Create a SystemMsg. hd.length is in 4-byte units. msg_text = "Market open notification" From 42568562b71bc6d791ab8b9b46504319f8d61e92 Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Mon, 25 May 2026 10:36:57 -0500 Subject: [PATCH 2/3] =?UTF-8?q?Bump=20version=200.1.0=20=E2=86=92=200.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-1.0 minor bump for the SymbolMappingMsg encoder fix: behavior of the encoder on cross-version writes changed (records that previously produced corrupt files now produce correct ones, and the on-disk hd.length of SymbolMappingMsg records is normalized to the file's metadata.version). Downstream consumers should opt in via their [compat] entry. Co-Authored-By: Claude Opus 4.7 (1M context) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 0a480c4..529cd10 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "DatabentoBinaryEncoding" uuid = "90689371-c8cb-40d1-831f-18033db90f74" authors = ["Tyler Beason "] -version = "0.1.0" +version = "0.2.0" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" From 1d36cfbd4789baf988ce238b23ec0d6a35a104bf Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Mon, 25 May 2026 10:41:27 -0500 Subject: [PATCH 3/3] Correct version bump to 0.1.1 (patch, not minor) Previous commit bumped to 0.2.0; the encoder fix is a bug fix without breaking API change, so patch is the right level. Downstream [compat] "0.1" entries pick it up automatically. Co-Authored-By: Claude Opus 4.7 (1M context) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 529cd10..e62118f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "DatabentoBinaryEncoding" uuid = "90689371-c8cb-40d1-831f-18033db90f74" authors = ["Tyler Beason "] -version = "0.2.0" +version = "0.1.1" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"