From ff6cea3e2b1414669d5bf51f7b3cfdc87a1d924f Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Mon, 18 May 2026 21:25:38 -0500 Subject: [PATCH 1/2] =?UTF-8?q?Rename=20package:=20DBN=20=E2=86=92=20Datab?= =?UTF-8?q?entoBinaryEncoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Julia Triage rejected the General Registry submission of "DBN" against three naming guidelines (all-caps, <5 chars, Damerau-Levenshtein distance ≤2 from DSP/BDF/JDBC/etc.). Rename to clear all three: - GitHub repo: DBN.jl → DatabentoBinaryEncoding.jl (already done) - src/DBN.jl → src/DatabentoBinaryEncoding.jl (file rename) - `module DBN` → `module DatabentoBinaryEncoding` - Project.toml: name = "DBN" → "DatabentoBinaryEncoding" (UUID unchanged, so downstream `[deps]` entries don't need re-mapping) - docs/make.jl: sitename, canonical URL, modules, repo all updated - README + docs/src/*.md prose: package-name references updated - benchmark/*: prose + the env-check error message updated Test/benchmark .jl files all use the alias pattern: using DatabentoBinaryEncoding import DatabentoBinaryEncoding as DBN This keeps all ~300 existing `DBN.foo` references unchanged — idiomatic Julia for verbose package names (cf. `LinearAlgebra as LA`). Local Julia 1.13 beta has a borked LibCURL that prevents Pkg.test(), but direct `julia --project=. test/runtests.jl` shows 3502 pass / 1 error (unchanged from pre-rename; the one error is the pre-existing BenchmarkTools-not-in-[deps] issue that only surfaces when bypassing Pkg.test()). CI uses Pkg.test() and should be all green. After this merges, retrigger registration via @JuliaRegistrator register and comment on JuliaRegistries/General#155767 with the link to the new registry PR. Co-Authored-By: Claude Opus 4.7 (1M context) --- Project.toml | 2 +- README.md | 11 +++++---- benchmark/BENCHMARK_RESULTS.md | 2 +- benchmark/EXAMPLE_OUTPUT.md | 8 +++---- benchmark/PERFORMANCE_REPORT.md | 10 ++++---- benchmark/README.md | 12 +++++----- benchmark/benchmarks.jl | 3 ++- benchmark/generate_test_data.jl | 3 ++- benchmark/run_benchmarks.jl | 16 +++++++------ benchmark/verify_setup.sh | 6 ++--- docs/make.jl | 10 ++++---- docs/src/api/enums.md | 4 ++-- docs/src/api/utilities.md | 6 ++--- docs/src/guide/conversion.md | 28 +++++++++++----------- docs/src/guide/reading.md | 14 +++++------ docs/src/guide/streaming.md | 12 +++++----- docs/src/guide/writing.md | 12 +++++----- docs/src/index.md | 16 ++++++------- docs/src/installation.md | 16 ++++++------- docs/src/performance.md | 14 +++++------ docs/src/quickstart.md | 12 +++++----- docs/src/troubleshooting.md | 18 +++++++------- src/{DBN.jl => DatabentoBinaryEncoding.jl} | 11 +++++---- test/compatibility_utils.jl | 3 ++- test/runtests.jl | 3 ++- test/test_compatibility_updated.jl | 3 ++- test/test_convenience_functions.jl | 3 ++- test/test_import_simple.jl | 3 ++- test/test_phase10_complete.jl | 3 ++- test/test_phase11_typed_with_control.jl | 3 ++- test/test_phase9_working.jl | 3 ++- 31 files changed, 142 insertions(+), 128 deletions(-) rename src/{DBN.jl => DatabentoBinaryEncoding.jl} (91%) diff --git a/Project.toml b/Project.toml index 73a19fa..0a480c4 100644 --- a/Project.toml +++ b/Project.toml @@ -1,4 +1,4 @@ -name = "DBN" +name = "DatabentoBinaryEncoding" uuid = "90689371-c8cb-40d1-831f-18033db90f74" authors = ["Tyler Beason "] version = "0.1.0" diff --git a/README.md b/README.md index b548ccb..fc48414 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DBN.jl +# DatabentoBinaryEncoding.jl Julia implementation of the Databento Binary Encoding (DBN) message encoding and storage format for normalized market data. @@ -19,7 +19,7 @@ For more details, read the [introduction to DBN](https://databento.com/docs/stan ## Performance -DBN.jl is optimized for high-throughput market data processing: +DatabentoBinaryEncoding.jl is optimized for high-throughput market data processing: - **Read**: Up to 40M records/sec with near-zero-allocation callback streaming - **Write**: 11M records/sec with optimized bulk operations - **Type-specific readers** (`read_trades`, `read_mbo`, etc.) are 5-6x faster than generic `read_dbn()` @@ -35,7 +35,7 @@ This package is not yet registered. Install directly from GitHub: ```julia using Pkg -Pkg.add(url="https://github.com/tbeason/DBN.jl") +Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") ``` ## Usage @@ -43,7 +43,8 @@ Pkg.add(url="https://github.com/tbeason/DBN.jl") ### Reading DBN Files ```julia -using DBN +using DatabentoBinaryEncoding +# Tip: `import DatabentoBinaryEncoding as DBN` gives you the terser `DBN.foo` prefix. # Read entire file into memory (fastest for bulk loading) records = read_dbn("trades.dbn") @@ -72,7 +73,7 @@ println("Total: $(total[])") ### Writing DBN Files ```julia -using DBN, Dates +using DatabentoBinaryEncoding, Dates # Create metadata for trades metadata = Metadata( diff --git a/benchmark/BENCHMARK_RESULTS.md b/benchmark/BENCHMARK_RESULTS.md index 489129f..c5facb1 100644 --- a/benchmark/BENCHMARK_RESULTS.md +++ b/benchmark/BENCHMARK_RESULTS.md @@ -1,5 +1,5 @@ ================================================================================ -DBN.jl COMPREHENSIVE PERFORMANCE COMPARISON +DatabentoBinaryEncoding.jl COMPREHENSIVE PERFORMANCE COMPARISON ================================================================================ diff --git a/benchmark/EXAMPLE_OUTPUT.md b/benchmark/EXAMPLE_OUTPUT.md index fc30a1c..67cc082 100644 --- a/benchmark/EXAMPLE_OUTPUT.md +++ b/benchmark/EXAMPLE_OUTPUT.md @@ -1,13 +1,13 @@ # Example Benchmark Output -This document shows example output from the DBN.jl benchmark suite. +This document shows example output from the DatabentoBinaryEncoding.jl benchmark suite. ## Throughput Benchmark Example ``` ████████████████████████████████████████████████████████████████████ █ █ -█ DBN.jl THROUGHPUT BENCHMARK SUITE █ +█ DatabentoBinaryEncoding.jl THROUGHPUT BENCHMARK SUITE █ █ █ ████████████████████████████████████████████████████████████████████ @@ -134,7 +134,7 @@ Completed at: 2024-01-15 10:45:23 ``` ====================================================================== -DBN.jl BenchmarkTools Suite +DatabentoBinaryEncoding.jl BenchmarkTools Suite ====================================================================== Creating benchmark suite... @@ -230,7 +230,7 @@ sys 0m0.012s ``` **Throughput:** ~5.05 million records/second -### Julia (DBN.jl) +### Julia (DatabentoBinaryEncoding.jl) ```julia julia> @time read_dbn("trades.1m.dbn.zst") 0.245000 seconds (450 allocations: 3.82 MiB) diff --git a/benchmark/PERFORMANCE_REPORT.md b/benchmark/PERFORMANCE_REPORT.md index b3ff908..26327c5 100644 --- a/benchmark/PERFORMANCE_REPORT.md +++ b/benchmark/PERFORMANCE_REPORT.md @@ -1,4 +1,4 @@ -# DBN.jl Performance Optimization Report +# DatabentoBinaryEncoding.jl Performance Optimization Report **Date**: 2025-10-26 **Branch**: `claude/benchmark-package-performance-011CUSRjJF6DMNNwX64UELMY` @@ -8,7 +8,7 @@ ## Executive Summary -This report documents the performance optimization work on DBN.jl, including critical bug fixes, code quality improvements, and performance benchmarking against the official Rust implementation. +This report documents the performance optimization work on DatabentoBinaryEncoding.jl, including critical bug fixes, code quality improvements, and performance benchmarking against the official Rust implementation. ### Key Achievements @@ -19,7 +19,7 @@ This report documents the performance optimization work on DBN.jl, including cri ### Current Performance: Julia vs Rust vs Python -| Metric | Julia DBN.jl | Rust dbn CLI | Python databento | Julia vs Rust | Julia vs Python | +| Metric | Julia DatabentoBinaryEncoding.jl | Rust dbn CLI | Python databento | Julia vs Rust | Julia vs Python | |--------|--------------|--------------|------------------|---------------|-----------------| | **Large files (1M+ records)** | 1.18-1.45 M rec/s | 2.22-2.68 M rec/s | 10.04-11.90 M rec/s | **1.77-1.84x slower** | **7.73-10.13x slower** | | **Medium files (100K records)** | 0.83-0.89 M rec/s | 1.78-1.94 M rec/s | 9.90-10.02 M rec/s | **2.04-2.17x slower** | **11.11-11.48x slower** | @@ -28,7 +28,7 @@ This report documents the performance optimization work on DBN.jl, including cri **Write Performance**: 2.0-2.2 M rec/s (excellent, competitive with Rust) -**Note on Python Performance**: The Python databento client uses Rust bindings (`databento-dbn`) under the hood, essentially providing a thin wrapper over the Rust implementation. This explains its excellent performance - it's doing minimal work in Python, with the heavy lifting done by compiled Rust code. This is a different architecture than Julia DBN.jl which implements the full decoding in Julia. +**Note on Python Performance**: The Python databento client uses Rust bindings (`databento-dbn`) under the hood, essentially providing a thin wrapper over the Rust implementation. This explains its excellent performance - it's doing minimal work in Python, with the heavy lifting done by compiled Rust code. This is a different architecture than Julia DatabentoBinaryEncoding.jl which implements the full decoding in Julia. --- @@ -289,7 +289,7 @@ end ## Conclusion -DBN.jl is currently **2.45x slower** than the Rust CLI implementation on average, which is a reasonable starting point for a Julia package. The write performance is already excellent at 2.0-2.2 M rec/s. +DatabentoBinaryEncoding.jl is currently **2.45x slower** than the Rust CLI implementation on average, which is a reasonable starting point for a Julia package. The write performance is already excellent at 2.0-2.2 M rec/s. **Key findings**: - ✅ Critical bugs fixed (compression, world age) diff --git a/benchmark/README.md b/benchmark/README.md index 2fd672e..d5fa510 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,6 +1,6 @@ -# DBN.jl Benchmark Suite +# DatabentoBinaryEncoding.jl Benchmark Suite -Comprehensive performance benchmarking tools for DBN.jl, measuring throughput (records/second), latency, memory usage, and bandwidth. +Comprehensive performance benchmarking tools for DatabentoBinaryEncoding.jl, measuring throughput (records/second), latency, memory usage, and bandwidth. ## Quick Start @@ -185,7 +185,7 @@ julia benchmark/run_benchmarks.jl --suite-only ### 4. Custom Analysis ```julia -using DBN +using DatabentoBinaryEncoding include("benchmark/throughput.jl") # Benchmark specific file @@ -262,7 +262,7 @@ For detailed profiling of hot spots: ```julia using Profile, ProfileView -using DBN +using DatabentoBinaryEncoding # Profile reading file = "benchmark/data/trades.1m.dbn" @@ -281,7 +281,7 @@ g = flamegraph() ## Memory Profiling ```julia -using DBN +using DatabentoBinaryEncoding # Track allocations file = "benchmark/data/trades.1m.dbn" @@ -327,7 +327,7 @@ julia benchmark/run_benchmarks.jl --throughput-only ## Contributing -When adding new features to DBN.jl: +When adding new features to DatabentoBinaryEncoding.jl: 1. Add relevant benchmarks to `benchmarks.jl` 2. Run `julia benchmark/run_benchmarks.jl --quick` to verify no regressions diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 2c9bcd2..ed269ee 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -8,7 +8,8 @@ min/median/max times, memory allocations, and garbage collection overhead. """ using BenchmarkTools -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using DataFrames using CSV using Printf diff --git a/benchmark/generate_test_data.jl b/benchmark/generate_test_data.jl index 0928940..8929cdf 100644 --- a/benchmark/generate_test_data.jl +++ b/benchmark/generate_test_data.jl @@ -7,7 +7,8 @@ This script creates DBN files of various sizes with realistic market data to enable thorough performance testing. """ -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Dates using Printf diff --git a/benchmark/run_benchmarks.jl b/benchmark/run_benchmarks.jl index 4c05f2b..7b82339 100644 --- a/benchmark/run_benchmarks.jl +++ b/benchmark/run_benchmarks.jl @@ -1,7 +1,7 @@ """ run_benchmarks.jl -Main entry point for running all DBN.jl benchmarks. +Main entry point for running all DatabentoBinaryEncoding.jl benchmarks. Usage: julia benchmark/run_benchmarks.jl [options] @@ -18,17 +18,19 @@ Options: using Pkg # Ensure we're in the right environment -# Check if DBN is either a dependency or if we're in the DBN.jl project itself +# Check if DatabentoBinaryEncoding is either a dependency or if we're in the +# DatabentoBinaryEncoding.jl project itself. proj = Pkg.project() -if !haskey(proj.dependencies, "DBN") && proj.name != "DBN" - error("DBN package not found. Make sure you're in the DBN.jl project directory.") +if !haskey(proj.dependencies, "DatabentoBinaryEncoding") && proj.name != "DatabentoBinaryEncoding" + error("DatabentoBinaryEncoding package not found. Make sure you're in the DatabentoBinaryEncoding.jl project directory.") end # Activate the project Pkg.activate(".") # Load DBN -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Printf using Dates @@ -98,7 +100,7 @@ Print usage information. """ function print_help() println(""" - DBN.jl Benchmark Suite + DatabentoBinaryEncoding.jl Benchmark Suite Usage: julia benchmark/run_benchmarks.jl [options] @@ -136,7 +138,7 @@ function print_banner() println("\n") println("█"^80) println("█" * " "^78 * "█") - println("█" * " DBN.jl - Databento Binary Encoding Performance Benchmark" * " "^12 * "█") + println("█" * " DatabentoBinaryEncoding.jl - Databento Binary Encoding Performance Benchmark" * " "^12 * "█") println("█" * " "^78 * "█") println("█"^80) println() diff --git a/benchmark/verify_setup.sh b/benchmark/verify_setup.sh index fc2a94f..6c21d33 100755 --- a/benchmark/verify_setup.sh +++ b/benchmark/verify_setup.sh @@ -2,7 +2,7 @@ # Verify benchmark suite setup echo "==================================================" -echo " DBN.jl Benchmark Suite Verification" +echo " DatabentoBinaryEncoding.jl Benchmark Suite Verification" echo "==================================================" echo @@ -35,7 +35,7 @@ if command -v julia &> /dev/null; then # Check if DBN can be loaded echo "Testing DBN package load..." - julia --project=. -e 'using DBN; println("✓ DBN.jl loaded successfully")' 2>&1 + julia --project=. -e 'using DatabentoBinaryEncoding; println("✓ DatabentoBinaryEncoding.jl loaded successfully")' 2>&1 if [ $? -eq 0 ]; then echo @@ -48,7 +48,7 @@ if command -v julia &> /dev/null; then echo else echo - echo "⚠ DBN.jl could not be loaded. Run:" + echo "⚠ DatabentoBinaryEncoding.jl could not be loaded. Run:" echo " julia --project=. -e 'using Pkg; Pkg.instantiate()'" echo fi diff --git a/docs/make.jl b/docs/make.jl index d82359f..dd3fd54 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,14 +1,14 @@ using Documenter -using DBN +using DatabentoBinaryEncoding makedocs( - sitename = "DBN.jl", + sitename = "DatabentoBinaryEncoding.jl", format = Documenter.HTML( prettyurls = get(ENV, "CI", nothing) == "true", - canonical = "https://tbeason.github.io/DBN.jl", + canonical = "https://tbeason.github.io/DatabentoBinaryEncoding.jl", assets = String[], ), - modules = [DBN], + modules = [DatabentoBinaryEncoding], pages = [ "Home" => "index.md", "Installation" => "installation.md", @@ -35,7 +35,7 @@ makedocs( ) deploydocs( - repo = "github.com/tbeason/DBN.jl.git", + repo = "github.com/tbeason/DatabentoBinaryEncoding.jl.git", devbranch = "main", push_preview = true, ) diff --git a/docs/src/api/enums.md b/docs/src/api/enums.md index 1f3f38c..84ac2c4 100644 --- a/docs/src/api/enums.md +++ b/docs/src/api/enums.md @@ -1,6 +1,6 @@ # Enums API Reference -Enumeration types used throughout DBN.jl. +Enumeration types used throughout DatabentoBinaryEncoding.jl. ## Schema Types @@ -115,7 +115,7 @@ Instrument classification: ### Working with Schemas ```julia -using DBN +using DatabentoBinaryEncoding # Check schema type if metadata.schema == Schema.TRADES diff --git a/docs/src/api/utilities.md b/docs/src/api/utilities.md index 66705a5..23c9791 100644 --- a/docs/src/api/utilities.md +++ b/docs/src/api/utilities.md @@ -14,7 +14,7 @@ float_to_price ### Usage Examples ```julia -using DBN +using DatabentoBinaryEncoding # Convert DBN fixed-point price to float fixed_price = 1005000 # 100.5000 in fixed-point @@ -49,7 +49,7 @@ to_nanoseconds ### Usage Examples ```julia -using DBN, Dates +using DatabentoBinaryEncoding, Dates # DateTime to nanoseconds dt = DateTime(2024, 1, 1, 9, 30, 0) @@ -111,7 +111,7 @@ DBN_VERSION::UInt8 = 3 # Supported DBN version DBN uses special sentinel values to indicate undefined/missing data: ```julia -using DBN +using DatabentoBinaryEncoding # Check for undefined price if trade.price == UNDEF_PRICE diff --git a/docs/src/guide/conversion.md b/docs/src/guide/conversion.md index 25a6a93..9f92d71 100644 --- a/docs/src/guide/conversion.md +++ b/docs/src/guide/conversion.md @@ -1,6 +1,6 @@ # Format Conversion -DBN.jl supports bidirectional conversion between DBN and other popular formats: CSV, JSON, Parquet, and DataFrames. +DatabentoBinaryEncoding.jl supports bidirectional conversion between DBN and other popular formats: CSV, JSON, Parquet, and DataFrames. ## Supported Formats @@ -16,7 +16,7 @@ DBN.jl supports bidirectional conversion between DBN and other popular formats: ### DBN to CSV ```julia -using DBN +using DatabentoBinaryEncoding # Convert DBN to CSV dbn_to_csv("trades.dbn", "trades.csv") @@ -42,7 +42,7 @@ ts_event,publisher_id,instrument_id,price,size,action,side,flags,depth,ts_recv,t ### DBN to JSON ```julia -using DBN +using DatabentoBinaryEncoding # Convert to JSON (array of objects) dbn_to_json("trades.dbn", "trades.json") @@ -84,7 +84,7 @@ dbn_to_json("trades.dbn", "trades.jsonl") ### DBN to Parquet ```julia -using DBN +using DatabentoBinaryEncoding # Convert to Parquet dbn_to_parquet("trades.dbn", "output_directory/") @@ -101,7 +101,7 @@ dbn_to_parquet("trades.dbn", "output_directory/") ### DBN to DataFrame ```julia -using DBN +using DatabentoBinaryEncoding # Read DBN file records = read_trades("trades.dbn") @@ -129,7 +129,7 @@ end ### JSON to DBN ```julia -using DBN +using DatabentoBinaryEncoding # Convert JSON to DBN json_to_dbn("trades.json", "trades.dbn") @@ -147,7 +147,7 @@ json_to_dbn("trades.jsonl", "trades.dbn") ### CSV to DBN ```julia -using DBN +using DatabentoBinaryEncoding # Convert CSV to DBN (requires schema specification) csv_to_dbn( @@ -175,7 +175,7 @@ csv_to_dbn( ### Parquet to DBN ```julia -using DBN +using DatabentoBinaryEncoding # Convert Parquet to DBN parquet_to_dbn( @@ -207,7 +207,7 @@ parquet_to_dbn( **Example: Export sample for Excel analysis** ```julia -using DBN +using DatabentoBinaryEncoding # Read first 10,000 trades trades = read_trades("large_file.dbn.zst") @@ -238,7 +238,7 @@ rm("sample.dbn") **Example: Export for web application** ```julia -using DBN +using DatabentoBinaryEncoding # Get recent trades trades = read_trades("recent.dbn") @@ -271,7 +271,7 @@ rm("latest.dbn") **Example: Export for DuckDB analysis** ```julia -using DBN +using DatabentoBinaryEncoding # Convert historical data to Parquet dbn_to_parquet("2024_trades.dbn.zst", "data_lake/") @@ -290,7 +290,7 @@ dbn_to_parquet("2024_trades.dbn.zst", "data_lake/") **Example: Analyze trade patterns** ```julia -using DBN, DataFrames, Statistics, Dates +using DatabentoBinaryEncoding, DataFrames, Statistics, Dates # Read data trades = read_trades("trades.dbn") @@ -318,7 +318,7 @@ end ### DBN → CSV → DBN ```julia -using DBN +using DatabentoBinaryEncoding # Original file metadata_original, records_original = read_dbn_with_metadata("original.dbn") @@ -397,7 +397,7 @@ dbn_to_csv("all.dbn.zst", "huge.csv") # Wasteful! ### 3. Batch Processing for Large Files ```julia # Process in chunks for very large files -using DBN +using DatabentoBinaryEncoding writer = open("output.csv", "w") println(writer, "ts_event,price,size,side") # Header diff --git a/docs/src/guide/reading.md b/docs/src/guide/reading.md index b0ad123..991f78a 100644 --- a/docs/src/guide/reading.md +++ b/docs/src/guide/reading.md @@ -1,6 +1,6 @@ # Reading Data -DBN.jl provides multiple ways to read DBN files, each optimized for different use cases. This guide helps you choose the right approach for your needs. +DatabentoBinaryEncoding.jl provides multiple ways to read DBN files, each optimized for different use cases. This guide helps you choose the right approach for your needs. ## Quick Reference @@ -43,7 +43,7 @@ Plus additional readers for other schemas. See the [API Reference](../api/readin **Example:** ```julia -using DBN +using DatabentoBinaryEncoding # Read trades file trades = read_trades("AAPL_trades_2024-01-01.dbn.zst") @@ -84,7 +84,7 @@ end **Example: Calculate VWAP** ```julia -using DBN +using DatabentoBinaryEncoding total_price_volume = Ref(0.0) total_volume = Ref(0) @@ -138,7 +138,7 @@ end **Example: Find First Record Matching Condition** ```julia -using DBN +using DatabentoBinaryEncoding function find_first_large_trade(filename, threshold) for trade in DBNStream(filename) @@ -175,7 +175,7 @@ metadata, records = read_dbn_with_metadata("file.dbn") **Example:** ```julia -using DBN +using DatabentoBinaryEncoding # Quick look at a file metadata, records = read_dbn_with_metadata("unknown.dbn") @@ -201,7 +201,7 @@ for record in DBNStream("data.dbn.zst") end ``` -No special handling needed - DBN.jl detects compression automatically! +No special handling needed - DatabentoBinaryEncoding.jl detects compression automatically! ## Reading Metadata @@ -300,7 +300,7 @@ end ## Error Handling ```julia -using DBN +using DatabentoBinaryEncoding try records = read_dbn("file.dbn") diff --git a/docs/src/guide/streaming.md b/docs/src/guide/streaming.md index 420b22d..815054d 100644 --- a/docs/src/guide/streaming.md +++ b/docs/src/guide/streaming.md @@ -1,6 +1,6 @@ # Streaming -Streaming is essential for working with large DBN files that don't fit in memory or when you want maximum performance. DBN.jl provides two streaming patterns: **callback-based** (fastest) and **iterator-based** (most flexible). +Streaming is essential for working with large DBN files that don't fit in memory or when you want maximum performance. DatabentoBinaryEncoding.jl provides two streaming patterns: **callback-based** (fastest) and **iterator-based** (most flexible). ## Why Stream? @@ -25,7 +25,7 @@ The callback pattern uses `foreach_*()` functions for maximum performance. ### Basic Usage ```julia -using DBN +using DatabentoBinaryEncoding # Process all trades with a callback total_volume = Ref(0) @@ -73,7 +73,7 @@ Bulk read: 6.86 M records/sec (1.457s, 687 MB allocated) #### Aggregation ```julia -using DBN +using DatabentoBinaryEncoding # Calculate VWAP price_volume_sum = Ref(0.0) @@ -192,7 +192,7 @@ The iterator pattern uses `DBNStream()` for maximum flexibility. ### Basic Usage ```julia -using DBN +using DatabentoBinaryEncoding # Iterate through records for record in DBNStream("file.dbn") @@ -282,7 +282,7 @@ end For mixed-schema files with callbacks: ```julia -using DBN +using DatabentoBinaryEncoding # Generic callback with type checking trade_count = Ref(0) @@ -421,7 +421,7 @@ Callback streaming uses **~6,000x less memory** than iterator streaming! ## Error Handling ```julia -using DBN +using DatabentoBinaryEncoding try foreach_trade("file.dbn") do trade diff --git a/docs/src/guide/writing.md b/docs/src/guide/writing.md index 6655267..dc6f0a3 100644 --- a/docs/src/guide/writing.md +++ b/docs/src/guide/writing.md @@ -1,6 +1,6 @@ # Writing Data -DBN.jl provides two main approaches for writing DBN files: bulk writing for pre-existing data, and streaming for real-time data ingestion. +DatabentoBinaryEncoding.jl provides two main approaches for writing DBN files: bulk writing for pre-existing data, and streaming for real-time data ingestion. ## Quick Reference @@ -16,7 +16,7 @@ DBN.jl provides two main approaches for writing DBN files: bulk writing for pre- Write a collection of records with metadata: ```julia -using DBN, Dates +using DatabentoBinaryEncoding, Dates # Your records (e.g., from reading another file or creating synthetically) records = [trade1, trade2, trade3, ...] @@ -125,7 +125,7 @@ For writing data as it arrives (real-time or sequential processing): ### Basic Usage ```julia -using DBN +using DatabentoBinaryEncoding # Create a streaming writer writer = DBNStreamWriter("output.dbn", "XNAS", Schema.TRADES) @@ -157,7 +157,7 @@ close_writer!(writer) ### Real-time Data Example ```julia -using DBN, Dates +using DatabentoBinaryEncoding, Dates # Create writer for live data writer = DBNStreamWriter("live_trades.dbn.zst", "XNAS", Schema.TRADES) @@ -198,7 +198,7 @@ end ### Trade Messages ```julia -using DBN, Dates +using DatabentoBinaryEncoding, Dates function create_trade(price::Float64, size::Int, side::Side.T, instrument_id::Int = 12345, @@ -393,7 +393,7 @@ write_dbn("output.dbn", metadata, records) ## Error Handling ```julia -using DBN +using DatabentoBinaryEncoding try writer = DBNStreamWriter("output.dbn", "XNAS", Schema.TRADES) diff --git a/docs/src/index.md b/docs/src/index.md index f165d02..1adabf0 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,4 +1,4 @@ -# DBN.jl Documentation +# DatabentoBinaryEncoding.jl Documentation Julia implementation of the Databento Binary Encoding (DBN) message encoding and storage format for normalized market data. @@ -14,9 +14,9 @@ DBN (Databento Binary Encoding) is a fast, compact binary format for storing and For details on the DBN format specification, see the [official Databento documentation](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). -## Why DBN.jl? +## Why DatabentoBinaryEncoding.jl? -DBN.jl brings the power of DBN to Julia with: +DatabentoBinaryEncoding.jl brings the power of DBN to Julia with: - ✅ **Complete DBN v3 Format Support** - All message types and schemas - ✅ **Efficient Streaming** - Read and write large files with minimal memory @@ -30,7 +30,7 @@ DBN.jl brings the power of DBN to Julia with: ## Quick Example ```julia -using DBN +using DatabentoBinaryEncoding # Read a DBN file trades = read_trades("trades.dbn") @@ -48,7 +48,7 @@ dbn_to_parquet("trades.dbn", "output_dir/") ## Performance Characteristics -DBN.jl is optimized for high-throughput market data processing: +DatabentoBinaryEncoding.jl is optimized for high-throughput market data processing: | Operation | Throughput | Method | |-----------|-----------|---------| @@ -73,7 +73,7 @@ See the [Performance](@ref) page for detailed benchmarks and optimization tips. ## Getting Started -1. [Install DBN.jl](installation.md) +1. [Install DatabentoBinaryEncoding.jl](installation.md) 2. Follow the [Quick Start Guide](quickstart.md) 3. Explore the [User Guide](guide/reading.md) 4. Check the [API Reference](api/reading.md) @@ -81,11 +81,11 @@ See the [Performance](@ref) page for detailed benchmarks and optimization tips. ## Getting Help - **Documentation**: You're reading it! Browse the sections in the sidebar -- **Issues**: Report bugs or request features on [GitHub](https://github.com/tbeason/DBN.jl/issues) +- **Issues**: Report bugs or request features on [GitHub](https://github.com/tbeason/DatabentoBinaryEncoding.jl/issues) - **Databento Docs**: For DBN format details, see [databento.com/docs](https://databento.com/docs/) ## License -DBN.jl is not affiliated with Databento. +DatabentoBinaryEncoding.jl is not affiliated with Databento. The official DBN implementations ([dbn](https://github.com/databento/dbn)) are distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). diff --git a/docs/src/installation.md b/docs/src/installation.md index 0b3fe77..4cb2efd 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -5,16 +5,16 @@ - Julia 1.12 or later - Operating System: Windows, macOS, or Linux -## Installing DBN.jl +## Installing DatabentoBinaryEncoding.jl !!! note - DBN.jl is not yet registered in the Julia General registry. Install directly from GitHub. + DatabentoBinaryEncoding.jl is not yet registered in the Julia General registry. Install directly from GitHub. ### From GitHub ```julia using Pkg -Pkg.add(url="https://github.com/tbeason/DBN.jl") +Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") ``` ### Development Installation @@ -23,17 +23,17 @@ If you want to modify the package or contribute: ```julia using Pkg -Pkg.develop(url="https://github.com/tbeason/DBN.jl") +Pkg.develop(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") ``` This will clone the repository to `~/.julia/dev/DBN`. ## Verifying Installation -Test that DBN.jl is installed correctly: +Test that DatabentoBinaryEncoding.jl is installed correctly: ```julia -using DBN +using DatabentoBinaryEncoding # Check package version println(Pkg.TOML.parsefile(joinpath(dirname(pathof(DBN)), "..", "Project.toml"))["version"]) @@ -45,7 +45,7 @@ println(Pkg.TOML.parsefile(joinpath(dirname(pathof(DBN)), "..", "Project.toml")) ## Dependencies -DBN.jl has the following dependencies (automatically installed): +DatabentoBinaryEncoding.jl has the following dependencies (automatically installed): - **CodecZstd** - Zstd compression support - **CSV** - CSV file conversion @@ -65,7 +65,7 @@ DBN.jl has the following dependencies (automatically installed): If you get a "Package not found" error, make sure you're using the full GitHub URL: ```julia -Pkg.add(url="https://github.com/tbeason/DBN.jl") +Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") ``` ### Dependency Conflicts diff --git a/docs/src/performance.md b/docs/src/performance.md index e738a38..6fceca7 100644 --- a/docs/src/performance.md +++ b/docs/src/performance.md @@ -1,6 +1,6 @@ # Performance -DBN.jl is designed for high-throughput market data processing. This page provides performance characteristics, benchmarks, and optimization tips. +DatabentoBinaryEncoding.jl is designed for high-throughput market data processing. This page provides performance characteristics, benchmarks, and optimization tips. ## Performance Summary @@ -201,24 +201,24 @@ end ## Comparison with Other Implementations -DBN.jl performance compared to official implementations: +DatabentoBinaryEncoding.jl performance compared to official implementations: ### Read Performance (Trades, Uncompressed) | Implementation | Size | Throughput | |----------------|------|------------| -| **DBN.jl (callback)** | 10M | 42.37 M/s | -| **DBN.jl (optimized)** | 10M | 27.94 M/s | +| **DatabentoBinaryEncoding.jl (callback)** | 10M | 42.37 M/s | +| **DatabentoBinaryEncoding.jl (optimized)** | 10M | 27.94 M/s | | Python databento-dbn | 10M | 9.23 M/s | | Rust dbn | - | (Reference) | -DBN.jl callback streaming is **4.5x faster** than Python implementation. +DatabentoBinaryEncoding.jl callback streaming is **4.5x faster** than Python implementation. ### Write Performance (Trades, Uncompressed) | Implementation | Size | Throughput | |----------------|------|------------| -| **DBN.jl** | 10M | 11.57 M/s | +| **DatabentoBinaryEncoding.jl** | 10M | 11.57 M/s | | Python databento-dbn | 10M | 43.79 M/s | Python has faster write performance due to optimized C extensions. @@ -236,7 +236,7 @@ Performance scales with: Run benchmarks on your hardware: ```julia -using DBN, BenchmarkTools +using DatabentoBinaryEncoding, BenchmarkTools # Download test data or generate synthetic data # trades = generate_test_trades(10_000_000) diff --git a/docs/src/quickstart.md b/docs/src/quickstart.md index 655e8ef..a7a8fe5 100644 --- a/docs/src/quickstart.md +++ b/docs/src/quickstart.md @@ -1,12 +1,12 @@ # Quick Start Guide -Get up and running with DBN.jl in 5 minutes! +Get up and running with DatabentoBinaryEncoding.jl in 5 minutes! ## Installation ```julia using Pkg -Pkg.add(url="https://github.com/tbeason/DBN.jl") +Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") ``` ## Basic Usage @@ -16,7 +16,7 @@ Pkg.add(url="https://github.com/tbeason/DBN.jl") The simplest way to read a DBN file: ```julia -using DBN +using DatabentoBinaryEncoding # Read entire file into memory records = read_dbn("trades.dbn") @@ -62,7 +62,7 @@ println("Total volume: $(total_volume[])") Create a DBN file from Julia data: ```julia -using DBN, Dates +using DatabentoBinaryEncoding, Dates # Create metadata metadata = Metadata( @@ -179,7 +179,7 @@ parquet_to_dbn("trades.parquet", "trades.dbn", ### Working with Compressed Files -DBN.jl transparently handles Zstd-compressed files: +DatabentoBinaryEncoding.jl transparently handles Zstd-compressed files: ```julia # Read compressed files (automatically detected by .zst extension) @@ -272,4 +272,4 @@ Now that you know the basics: - **Troubleshooting**: See [common issues](troubleshooting.md) - **Databento Format**: Read the [official DBN documentation](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) - **Schemas**: Learn about [DBN schemas](https://databento.com/docs/schemas-and-data-formats) -- **Bug Reports**: Open an issue on [GitHub](https://github.com/tbeason/DBN.jl/issues) +- **Bug Reports**: Open an issue on [GitHub](https://github.com/tbeason/DatabentoBinaryEncoding.jl/issues) diff --git a/docs/src/troubleshooting.md b/docs/src/troubleshooting.md index 270c45d..5959293 100644 --- a/docs/src/troubleshooting.md +++ b/docs/src/troubleshooting.md @@ -1,6 +1,6 @@ # Troubleshooting -Common issues and solutions when working with DBN.jl. +Common issues and solutions when working with DatabentoBinaryEncoding.jl. ## Installation Issues @@ -8,10 +8,10 @@ Common issues and solutions when working with DBN.jl. **Problem**: `ERROR: Package DBN not found` -**Solution**: DBN.jl is not registered. Install from GitHub: +**Solution**: DatabentoBinaryEncoding.jl is not registered. Install from GitHub: ```julia using Pkg -Pkg.add(url="https://github.com/tbeason/DBN.jl") +Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") ``` ### Dependency Conflicts @@ -29,7 +29,7 @@ If problems persist: ```julia Pkg.rm("DBN") Pkg.gc() # Clean up -Pkg.add(url="https://github.com/tbeason/DBN.jl") +Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") ``` ## Compression Issues @@ -147,7 +147,7 @@ end 3. **Check for type instability**: ```julia # Run with type checking -using DBN +using DatabentoBinaryEncoding @code_warntype read_trades("file.dbn") ``` @@ -255,7 +255,7 @@ record.ts_event = DateTime(2024, 1, 1) # WRONG! **Problem**: `ERROR: Unsupported DBN version 1` -**Cause**: DBN.jl only supports DBN v2 and v3 +**Cause**: DatabentoBinaryEncoding.jl only supports DBN v2 and v3 **Solution**: Upgrade v1 files using Databento CLI: ```bash @@ -363,11 +363,11 @@ names(df) # Should match DBN field names If you encounter issues not covered here: 1. **Check the documentation** for the specific function -2. **Search existing GitHub issues**: [github.com/tbeason/DBN.jl/issues](https://github.com/tbeason/DBN.jl/issues) +2. **Search existing GitHub issues**: [github.com/tbeason/DatabentoBinaryEncoding.jl/issues](https://github.com/tbeason/DatabentoBinaryEncoding.jl/issues) 3. **Ask on GitHub Discussions** (if available) 4. **Open a new issue** with: - Julia version (`versioninfo()`) - - DBN.jl version + - DatabentoBinaryEncoding.jl version - Minimal example reproducing the issue - Error message and stack trace @@ -408,5 +408,5 @@ result = read_trades("test.dbn") # Error occurs here - [Quick Start Guide](quickstart.md) - Basic usage - [Reading Guide](guide/reading.md) - Reading methods - [Performance Guide](performance.md) - Optimization -- [GitHub Issues](https://github.com/tbeason/DBN.jl/issues) - Report bugs +- [GitHub Issues](https://github.com/tbeason/DatabentoBinaryEncoding.jl/issues) - Report bugs - [Databento Documentation](https://databento.com/docs/) - DBN format details diff --git a/src/DBN.jl b/src/DatabentoBinaryEncoding.jl similarity index 91% rename from src/DBN.jl rename to src/DatabentoBinaryEncoding.jl index 31e2047..41adb2d 100644 --- a/src/DBN.jl +++ b/src/DatabentoBinaryEncoding.jl @@ -1,11 +1,11 @@ """ - DBN + DatabentoBinaryEncoding Julia implementation of the Databento Binary Encoding (DBN) format for normalized market data. # Overview -DBN.jl provides complete support for reading and writing DBN v3 format files with: +DatabentoBinaryEncoding.jl provides complete support for reading and writing DBN v3 format files with: - Efficient streaming support for large files - Automatic Zstd compression/decompression - All DBN v3 message types @@ -44,7 +44,8 @@ DBN.jl provides complete support for reading and writing DBN v3 format files wit # Example Usage ```julia -using DBN +using DatabentoBinaryEncoding +# Or, for terser internal references: `import DatabentoBinaryEncoding as DBN` # Reading data records = read_dbn("data.dbn") @@ -75,7 +76,7 @@ close_writer!(writer) See the [DBN specification](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) for complete format documentation. """ -module DBN +module DatabentoBinaryEncoding # All using statements at the top using Dates @@ -123,4 +124,4 @@ export write_header, read_header!, write_record, read_record, finalize_encoder export dbn_to_csv, dbn_to_json, dbn_to_parquet, records_to_dataframe export json_to_dbn, parquet_to_dbn, csv_to_dbn -end # module DBN \ No newline at end of file +end # module DatabentoBinaryEncoding \ No newline at end of file diff --git a/test/compatibility_utils.jl b/test/compatibility_utils.jl index 999bfe9..02034c7 100644 --- a/test/compatibility_utils.jl +++ b/test/compatibility_utils.jl @@ -1,6 +1,7 @@ module CompatibilityUtils -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Test using JSON3 using StructTypes diff --git a/test/runtests.jl b/test/runtests.jl index e8f9212..a02afe7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using Test -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Dates # Load test utilities (safe_rm, etc.) diff --git a/test/test_compatibility_updated.jl b/test/test_compatibility_updated.jl index 9c04466..ad93c16 100644 --- a/test/test_compatibility_updated.jl +++ b/test/test_compatibility_updated.jl @@ -1,5 +1,6 @@ using Test -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Dates using JSON3 diff --git a/test/test_convenience_functions.jl b/test/test_convenience_functions.jl index 486feca..ae50aa0 100644 --- a/test/test_convenience_functions.jl +++ b/test/test_convenience_functions.jl @@ -1,5 +1,6 @@ using Test -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Dates @testset "Convenience Functions" begin diff --git a/test/test_import_simple.jl b/test/test_import_simple.jl index 392d73d..e3c78ea 100644 --- a/test/test_import_simple.jl +++ b/test/test_import_simple.jl @@ -1,5 +1,6 @@ using Test -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Dates @testset "Basic Import Functionality" begin diff --git a/test/test_phase10_complete.jl b/test/test_phase10_complete.jl index ff5509c..72c3338 100644 --- a/test/test_phase10_complete.jl +++ b/test/test_phase10_complete.jl @@ -1,5 +1,6 @@ using Test -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Dates using Statistics using BenchmarkTools diff --git a/test/test_phase11_typed_with_control.jl b/test/test_phase11_typed_with_control.jl index 2442282..be9cf73 100644 --- a/test/test_phase11_typed_with_control.jl +++ b/test/test_phase11_typed_with_control.jl @@ -3,7 +3,8 @@ # reader to split data records from gateway control records. using Test -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN @testset "foreach_record_with_control" begin diff --git a/test/test_phase9_working.jl b/test/test_phase9_working.jl index f2bce87..4f7cd44 100644 --- a/test/test_phase9_working.jl +++ b/test/test_phase9_working.jl @@ -1,5 +1,6 @@ using Test -using DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN using Dates @testset "Phase 9: Edge Cases and Error Handling" begin From a1761ee9d4509f79f6e0f27ef2a681d386c702ab Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Mon, 18 May 2026 21:28:10 -0500 Subject: [PATCH 2/2] =?UTF-8?q?docs:=20update=20Project.toml=20dep=20key?= =?UTF-8?q?=20DBN=20=E2=86=92=20DatabentoBinaryEncoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's docs/deploy build failed with "Refusing to add package DatabentoBinaryEncoding [...]. Package DBN=... with the same UUID already exists as a direct dependency." — caught by the rename PR because docs/Project.toml still declared the dep under the old name while the renamed package now reports as DatabentoBinaryeEncoding. UUID unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index d3ccb36..723774a 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,3 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -DBN = "90689371-c8cb-40d1-831f-18033db90f74" +DatabentoBinaryEncoding = "90689371-c8cb-40d1-831f-18033db90f74"