diff --git a/Project.toml b/Project.toml index 0a480c4..e62118f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "DatabentoBinaryEncoding" uuid = "90689371-c8cb-40d1-831f-18033db90f74" authors = ["Tyler Beason "] -version = "0.1.0" +version = "0.1.1" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" diff --git a/src/encode.jl b/src/encode.jl index f7f3095..07d22c7 100644 --- a/src/encode.jl +++ b/src/encode.jl @@ -791,11 +791,21 @@ function write_record_complex(encoder::DBNEncoder, record) elseif isa(record, SymbolMappingMsg) # Spec-compliant SymbolMappingMsg layout (depends on DBN version): - # v1: stype_in_symbol[22] | stype_out_symbol[22] | pad(4) | start_ts(8) | end_ts(8) - # (no explicit stype_in/stype_out bytes in v1) + # v1: stype_in_symbol[22] | stype_out_symbol[22] | pad(4) | start_ts(8) | end_ts(8) (body 64, total 80, hd.length 20) # v2+: stype_in(1) | stype_in_symbol[71] | stype_out(1) | stype_out_symbol[71] | - # start_ts(8) | end_ts(8) - write_record_header(io, record.hd) + # start_ts(8) | end_ts(8) (body 160, total 176, hd.length 44) + # + # The record's hd.length may reflect a different on-wire version than + # the file we're writing into — the Databento Live gateway emits v1 + # layout (hd.length = 20) even when the consumer is writing a v3 file. + # Re-derive the length from the layout we are about to write so the + # resulting record header matches the bytes that follow it. + body_bytes = encoder.metadata.version == 1 ? 64 : 160 + fixed_length = UInt8((16 + body_bytes) ÷ LENGTH_MULTIPLIER) + hd = record.hd + out_hd = hd.length == fixed_length ? hd : + RecordHeader(fixed_length, hd.rtype, hd.publisher_id, hd.instrument_id, hd.ts_event) + write_record_header(io, out_hd) if encoder.metadata.version == 1 _write_fixed_string(io, record.stype_in_symbol, 22) _write_fixed_string(io, record.stype_out_symbol, 22) diff --git a/test/test_phase8.jl b/test/test_phase8.jl index c492fef..1a66e9c 100644 --- a/test/test_phase8.jl +++ b/test/test_phase8.jl @@ -247,10 +247,56 @@ using Dates @test records[1].stype_in_symbol == "AAPL.NASDAQ" @test records[1].stype_out == SType.INSTRUMENT_ID @test records[1].stype_out_symbol == "12345" - + rm(test_file, force=true) end - + + @testset "SymbolMappingMsg cross-version (v1 wire → v3 file)" begin + # Live Databento gateway emits SymbolMappingMsg in v1 wire layout + # (hd.length=20, 80 bytes total) even when the consumer writes a + # v3 file. The encoder must re-derive the header length from the + # v2+ layout it is actually about to serialize (hd.length=44, + # 176 bytes total) so that the resulting record header matches + # the body and the file remains parseable. + v1_wire_length = UInt8(20) + mapping = SymbolMappingMsg( + RecordHeader(v1_wire_length, RType.SYMBOL_MAPPING_MSG, + UInt16(0), UInt32(1191182337), 1779203015494476543), + SType.INSTRUMENT_ID, + "SPX.OPT", + SType.INSTRUMENT_ID, + "SPX 271217C02800000", + Int64(-1), + Int64(-1), + ) + + # Pair with a follow-up record so we exercise the stream offset: + # if hd.length on the SymbolMappingMsg is wrong, the decoder + # mis-positions for the next read and the file is unparseable. + # TradeMsg on-wire: 16-byte header + 32-byte body = 48 bytes, hd.length = 12 + trade = TradeMsg( + RecordHeader(UInt8(12), RType.MBP_0_MSG, UInt16(0), UInt32(1191182337), + 1779203015494476600), + Int64(100_000_000_000), UInt32(10), + Action.TRADE, Side.BID, 0x00, 0x00, + Int64(1779203015494476600), Int32(0), UInt32(1), + ) + + write_dbn(test_file, metadata, [mapping, trade]) + records = read_dbn(test_file) + + @test length(records) == 2 + @test records[1] isa SymbolMappingMsg + @test records[1].stype_in_symbol == "SPX.OPT" + @test records[1].stype_out_symbol == "SPX 271217C02800000" + @test records[1].hd.instrument_id == UInt32(1191182337) + @test records[1].hd.length == UInt8(44) # rewritten to v3 layout + @test records[2] isa TradeMsg + @test records[2].hd.instrument_id == UInt32(1191182337) + + rm(test_file, force=true) + end + @testset "SystemMsg Write/Read" begin # Create a SystemMsg. hd.length is in 4-byte units. msg_text = "Market open notification"