Skip to content

Fix DBN v3 instrument definition layout #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tbeason merged 1 commit into from
Jun 1, 2026
Merged

Fix DBN v3 instrument definition layout #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 34 additions & 45 deletions src/decode.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -955,7 +955,7 @@ end
currency = intern_string(decoder, read(decoder.io, 4)) # offset 170
settl_currency = intern_string(decoder, read(decoder.io, 4)) # offset 174
secsubtype = intern_string(decoder, read(decoder.io, 6)) # offset 178
raw_symbol = intern_string(decoder, read(decoder.io, 71)) # offset 184 (71 bytes in v2!)
raw_symbol = intern_string(decoder, read(decoder.io, SYMBOL_CSTR_LEN)) # offset 184
group = intern_string(decoder, read(decoder.io, 21)) # offset 255
exchange = intern_string(decoder, read(decoder.io, 5)) # offset 276
asset = intern_string(decoder, read(decoder.io, 7)) # offset 281 (7 bytes in v2!)
Expand Down Expand Up @@ -996,10 +996,10 @@ end
leg_raw_symbol = ""
leg_instrument_class = InstrumentClass.OTHER # Default is OTHER, not UNKNOWN_0
leg_side = Side.NONE
leg_ratio_price_numerator = UInt32(0)
leg_ratio_price_denominator = UInt32(0)
leg_ratio_qty_numerator = UInt32(0)
leg_ratio_qty_denominator = UInt32(0)
leg_ratio_price_numerator = Int32(0)
leg_ratio_price_denominator = Int32(0)
leg_ratio_qty_numerator = Int32(0)
leg_ratio_qty_denominator = Int32(0)
leg_underlying_id = UInt32(0)

return InstrumentDefMsg(
Expand All @@ -1026,44 +1026,9 @@ end

@inline function read_instrument_def_v3(decoder::DBNDecoder, hd::RecordHeader)
# ===== DBN V3 InstrumentDefMsg =====
# DBN binary records are encoded in the fixed binary field layout.
# `encode_order` controls text/field ordering, not the binary layout.
ts_recv = read(decoder.io, Int64)
# encode_order 2: raw_symbol (22 bytes in v3)
raw_symbol = read_null_terminated_string(decoder.io, 22)
# encode_order 3: security_update_action
security_update_action_byte = read(decoder.io, UInt8)
security_update_action = security_update_action_byte == 0 ? '\0' : Char(security_update_action_byte)
# encode_order 4: instrument_class
instrument_class_byte = read(decoder.io, UInt8)
instrument_class = safe_instrument_class(instrument_class_byte)
# encode_order 20: raw_instrument_id (u64 in v3)
raw_instrument_id = read(decoder.io, UInt64)
# encode_order 54: strike_price
strike_price = read(decoder.io, Int64)
# encode_order 158: leg_count
leg_count = read(decoder.io, UInt16)
# encode_order 159: leg_index
leg_index = read(decoder.io, UInt16)
# encode_order 160: leg_instrument_id
leg_instrument_id = read(decoder.io, UInt32)
# encode_order 161: leg_raw_symbol (20 bytes)
leg_raw_symbol = intern_string(decoder, read(decoder.io, 20))
# encode_order 163: leg_instrument_class
leg_instrument_class_byte = read(decoder.io, UInt8)
leg_instrument_class = safe_instrument_class(leg_instrument_class_byte)
# encode_order 164: leg_side
leg_side_byte = read(decoder.io, UInt8)
leg_side = unsafe_side(leg_side_byte)
# encode_order 165: leg_price
leg_price = read(decoder.io, Int64)
# encode_order 166: leg_delta
leg_delta = read(decoder.io, Int64)
# encode_order 167-171: leg ratio fields
leg_ratio_price_numerator = read(decoder.io, UInt32)
leg_ratio_price_denominator = read(decoder.io, UInt32)
leg_ratio_qty_numerator = read(decoder.io, UInt32)
leg_ratio_qty_denominator = read(decoder.io, UInt32)
leg_underlying_id = read(decoder.io, UInt32)
# Now all fields WITHOUT encode_order, in struct declaration order
min_price_increment = read(decoder.io, Int64)
display_factor = read(decoder.io, Int64)
expiration = read(decoder.io, Int64)
Expand All @@ -1074,6 +1039,10 @@ end
unit_of_measure_qty = read(decoder.io, Int64)
min_price_increment_amount = read(decoder.io, Int64)
price_ratio = read(decoder.io, Int64)
strike_price = read(decoder.io, Int64)
raw_instrument_id = read(decoder.io, UInt64)
leg_price = read(decoder.io, Int64)
leg_delta = read(decoder.io, Int64)
inst_attrib_value = read(decoder.io, Int32)
underlying_id = read(decoder.io, UInt32)
market_depth_implied = read(decoder.io, Int32)
Expand All @@ -1087,15 +1056,24 @@ end
contract_multiplier = read(decoder.io, Int32)
decay_quantity = read(decoder.io, Int32)
original_contract_size = read(decoder.io, Int32)
leg_instrument_id = read(decoder.io, UInt32)
leg_ratio_price_numerator = read(decoder.io, Int32)
leg_ratio_price_denominator = read(decoder.io, Int32)
leg_ratio_qty_numerator = read(decoder.io, Int32)
leg_ratio_qty_denominator = read(decoder.io, Int32)
leg_underlying_id = read(decoder.io, UInt32)
appl_id = read(decoder.io, Int16)
maturity_year = read(decoder.io, UInt16)
decay_start_date = read(decoder.io, UInt16)
channel_id = read(decoder.io, UInt16)
# String fields without encode_order
# Use string interning for repeated values (currencies, exchanges, etc.)
leg_count = read(decoder.io, UInt16)
leg_index = read(decoder.io, UInt16)

# String fields in struct declaration order.
currency = intern_string(decoder, read(decoder.io, 4))
settl_currency = intern_string(decoder, read(decoder.io, 4))
secsubtype = intern_string(decoder, read(decoder.io, 6))
raw_symbol = intern_string(decoder, read(decoder.io, SYMBOL_CSTR_LEN))
group = intern_string(decoder, read(decoder.io, 21))
exchange = intern_string(decoder, read(decoder.io, 5))
asset = intern_string(decoder, read(decoder.io, 11)) # 11 bytes in v3!
Expand All @@ -1104,13 +1082,19 @@ end
unit_of_measure = intern_string(decoder, read(decoder.io, 31))
underlying = intern_string(decoder, read(decoder.io, 21))
strike_price_currency = intern_string(decoder, read(decoder.io, 4))
leg_raw_symbol = intern_string(decoder, read(decoder.io, SYMBOL_CSTR_LEN))

# Single-byte fields without encode_order
instrument_class_byte = read(decoder.io, UInt8)
instrument_class = safe_instrument_class(instrument_class_byte)
match_algorithm_byte = read(decoder.io, UInt8)
match_algorithm = match_algorithm_byte == 0 ? '\0' : Char(match_algorithm_byte)
main_fraction = read(decoder.io, UInt8)
price_display_format = read(decoder.io, UInt8)
sub_fraction = read(decoder.io, UInt8)
underlying_product = read(decoder.io, UInt8)
security_update_action_byte = read(decoder.io, UInt8)
security_update_action = security_update_action_byte == 0 ? '\0' : Char(security_update_action_byte)
maturity_month = read(decoder.io, UInt8)
maturity_day = read(decoder.io, UInt8)
maturity_week = read(decoder.io, UInt8)
Expand All @@ -1119,6 +1103,11 @@ end
contract_multiplier_unit = read(decoder.io, Int8)
flow_schedule_type = read(decoder.io, Int8)
tick_rule = read(decoder.io, UInt8)
leg_instrument_class_byte = read(decoder.io, UInt8)
leg_instrument_class = safe_instrument_class(leg_instrument_class_byte)
leg_side_byte = read(decoder.io, UInt8)
leg_side = unsafe_side(leg_side_byte)

# v3: 17 bytes _reserved
skip(decoder.io, 17)
# V3 has NO v2-only fields - set to defaults
Expand Down Expand Up @@ -1664,4 +1653,4 @@ read_bbo1s(filename::String) = read_dbn_typed(filename, BBO1sMsg)

Fast reader for BBO 1-minute data files. 5-6x faster than `read_dbn()`.
"""
read_bbo1m(filename::String) = read_dbn_typed(filename, BBO1mMsg)
read_bbo1m(filename::String) = read_dbn_typed(filename, BBO1mMsg)
169 changes: 70 additions & 99 deletions src/encode.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ function write_header(encoder::DBNEncoder)
write(metadata_buf, encoder.metadata.ts_out ? UInt8(1) : UInt8(0))

# Symbol string length (2 bytes) - only for version > 1
# DBN v3 uses 71-byte symbol length (same as v2)
symbol_cstr_len = UInt16(71)
# DBN v2/v3 use 71-byte symbol strings.
symbol_cstr_len = UInt16(SYMBOL_CSTR_LEN)
write(metadata_buf, htol(symbol_cstr_len))

# Reserved padding (53 bytes for v3)
Expand Down Expand Up @@ -498,71 +498,28 @@ function write_record_complex(encoder::DBNEncoder, record)
write(io, UInt8(0))
end

else # v3
# ===== DBN V3 InstrumentDefMsg =====
# encode_order 0: ts_recv
write(io, record.ts_recv)

# encode_order 2: raw_symbol (22 bytes in v3)
write_fixed_string(io, record.raw_symbol, 22)

# encode_order 3: security_update_action
write(io, UInt8(record.security_update_action))

# encode_order 4: instrument_class
write(io, UInt8(record.instrument_class))

# encode_order 20: raw_instrument_id (u64 in v3)
write(io, record.raw_instrument_id)

# encode_order 54: strike_price
write(io, record.strike_price)

# encode_order 158: leg_count
write(io, record.leg_count)

# encode_order 159: leg_index
write(io, record.leg_index)

# encode_order 160: leg_instrument_id
write(io, record.leg_instrument_id)

# encode_order 161: leg_raw_symbol (20 bytes)
write_fixed_string(io, record.leg_raw_symbol, 20)

# encode_order 163: leg_instrument_class
write(io, UInt8(record.leg_instrument_class))

# encode_order 164: leg_side
write(io, UInt8(record.leg_side))

# encode_order 165: leg_price
write(io, record.leg_price)

# encode_order 166: leg_delta
write(io, record.leg_delta)

# encode_order 167-171: leg ratio fields
write(io, record.leg_ratio_price_numerator)
write(io, record.leg_ratio_price_denominator)
write(io, record.leg_ratio_qty_numerator)
write(io, record.leg_ratio_qty_denominator)
write(io, record.leg_underlying_id)

# Now all fields WITHOUT encode_order, in struct declaration order
write(io, record.min_price_increment)
write(io, record.display_factor)
write(io, record.expiration)
else # v3
# ===== DBN V3 InstrumentDefMsg =====
# DBN binary records are encoded in the fixed binary field layout.
# `encode_order` controls text/field ordering, not the binary layout.
write(io, record.ts_recv)
write(io, record.min_price_increment)
write(io, record.display_factor)
write(io, record.expiration)
write(io, record.activation)
write(io, record.high_limit_price)
write(io, record.low_limit_price)
write(io, record.max_price_variation)
write(io, record.unit_of_measure_qty)
write(io, record.min_price_increment_amount)
write(io, record.price_ratio)

write(io, record.inst_attrib_value)
write(io, record.underlying_id)
write(io, record.unit_of_measure_qty)
write(io, record.min_price_increment_amount)
write(io, record.price_ratio)
write(io, record.strike_price)
write(io, record.raw_instrument_id)
write(io, record.leg_price)
write(io, record.leg_delta)

write(io, record.inst_attrib_value)
write(io, record.underlying_id)
write(io, record.market_depth_implied)
write(io, record.market_depth)
write(io, record.market_segment_id)
Expand All @@ -571,44 +528,58 @@ function write_record_complex(encoder::DBNEncoder, record)
write(io, record.min_lot_size_block)
write(io, record.min_lot_size_round_lot)
write(io, record.min_trade_vol)
write(io, record.contract_multiplier)
write(io, record.decay_quantity)
write(io, record.original_contract_size)

write(io, record.appl_id)
write(io, record.maturity_year)
write(io, record.decay_start_date)
write(io, record.channel_id)

# String fields without encode_order
write_fixed_string(io, record.currency, 4)
write_fixed_string(io, record.settl_currency, 4)
write_fixed_string(io, record.secsubtype, 6)
write_fixed_string(io, record.group, 21)
write_fixed_string(io, record.exchange, 5)
write_fixed_string(io, record.asset, 11) # 11 bytes in v3!
write(io, record.contract_multiplier)
write(io, record.decay_quantity)
write(io, record.original_contract_size)
write(io, record.leg_instrument_id)
write(io, record.leg_ratio_price_numerator)
write(io, record.leg_ratio_price_denominator)
write(io, record.leg_ratio_qty_numerator)
write(io, record.leg_ratio_qty_denominator)
write(io, record.leg_underlying_id)

write(io, record.appl_id)
write(io, record.maturity_year)
write(io, record.decay_start_date)
write(io, record.channel_id)
write(io, record.leg_count)
write(io, record.leg_index)

# String fields in struct declaration order.
write_fixed_string(io, record.currency, 4)
write_fixed_string(io, record.settl_currency, 4)
write_fixed_string(io, record.secsubtype, 6)
write_fixed_string(io, record.raw_symbol, SYMBOL_CSTR_LEN)
write_fixed_string(io, record.group, 21)
write_fixed_string(io, record.exchange, 5)
write_fixed_string(io, record.asset, 11) # 11 bytes in v3!
write_fixed_string(io, record.cfi, 7)
write_fixed_string(io, record.security_type, 7)
write_fixed_string(io, record.unit_of_measure, 31)
write_fixed_string(io, record.underlying, 21)
write_fixed_string(io, record.strike_price_currency, 4)

# Single-byte fields without encode_order
write(io, UInt8(record.match_algorithm))
write(io, record.main_fraction)
write(io, record.price_display_format)
write(io, record.sub_fraction)
write(io, record.underlying_product)
write(io, record.maturity_month)
write(io, record.maturity_day)
write(io, record.maturity_week)
write_fixed_string(io, record.unit_of_measure, 31)
write_fixed_string(io, record.underlying, 21)
write_fixed_string(io, record.strike_price_currency, 4)
write_fixed_string(io, record.leg_raw_symbol, SYMBOL_CSTR_LEN)

# Single-byte fields without encode_order
write(io, UInt8(record.instrument_class))
write(io, UInt8(record.match_algorithm))
write(io, record.main_fraction)
write(io, record.price_display_format)
write(io, record.sub_fraction)
write(io, record.underlying_product)
write(io, UInt8(record.security_update_action))
write(io, record.maturity_month)
write(io, record.maturity_day)
write(io, record.maturity_week)
write(io, record.user_defined_instrument ? UInt8('Y') : UInt8('N'))
write(io, record.contract_multiplier_unit)
write(io, record.flow_schedule_type)
write(io, record.tick_rule)

# v3: 17 bytes _reserved
for _ in 1:17
write(io, record.contract_multiplier_unit)
write(io, record.flow_schedule_type)
write(io, record.tick_rule)
write(io, UInt8(record.leg_instrument_class))
write(io, UInt8(record.leg_side))

# v3: 17 bytes _reserved
for _ in 1:17
write(io, UInt8(0))
end
end
Expand Down Expand Up @@ -933,4 +904,4 @@ function write_dbn(filename::String, metadata::Metadata, records)
close(base_io)
end
end
end
end
Loading
Loading