From ca21a018f4b01f2f35be45961bf199f4f4f6fcf5 Mon Sep 17 00:00:00 2001 From: Manfred Lee Date: Fri, 13 Mar 2026 16:55:59 -0700 Subject: [PATCH 1/6] Add mongodb --- Cargo.lock | 435 ++++++++++++++++++++- Cargo.toml | 5 +- README.md | 6 +- example/configs/mongo-to-iceberg.json | 30 ++ example/configs/mongo-to-kafka.json | 18 + example/configs/mongo-to-pg.json | 23 ++ example/configs/mongo-to-stdout.json | 16 + example/docker-compose.yml | 22 ++ example/init/mongodb/init.js | 41 ++ src/config/mod.rs | 4 + src/config/mongodb_source.rs | 66 ++++ src/config/source.rs | 21 + src/error.rs | 3 + src/lib.rs | 2 +- src/main.rs | 23 +- src/schema/discovery.rs | 88 +++++ src/schema/mod.rs | 2 + src/schema/type_mapping.rs | 57 +++ src/sink/postgres/mod.rs | 20 +- src/source/mod.rs | 1 + src/source/mongodb/bson_mapping.rs | 271 +++++++++++++ src/source/mongodb/change_stream.rs | 110 ++++++ src/source/mongodb/event_converter.rs | 128 ++++++ src/source/mongodb/mod.rs | 95 +++++ src/source/mongodb/snapshot.rs | 107 +++++ tests/mongodb_integration.rs | 538 ++++++++++++++++++++++++++ 26 files changed, 2117 insertions(+), 15 deletions(-) create mode 100644 example/configs/mongo-to-iceberg.json create mode 100644 example/configs/mongo-to-kafka.json create mode 100644 example/configs/mongo-to-pg.json create mode 100644 example/configs/mongo-to-stdout.json create mode 100644 example/init/mongodb/init.js create mode 100644 src/config/mongodb_source.rs create mode 100644 src/source/mongodb/bson_mapping.rs create mode 100644 src/source/mongodb/change_stream.rs create mode 100644 src/source/mongodb/event_converter.rs create mode 100644 src/source/mongodb/mod.rs create mode 100644 src/source/mongodb/snapshot.rs create mode 100644 tests/mongodb_integration.rs diff --git a/Cargo.lock b/Cargo.lock index 5ec5631..aa26b54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,6 +681,29 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash 0.8.12", + "base64 0.22.1", + "bitvec", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap 2.13.0", + "js-sys", + "once_cell", + "rand 0.9.2", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "btoi" version = "0.4.3" @@ -763,6 +786,7 @@ dependencies = [ "futures-util", "iceberg", "iceberg-catalog-rest", + "mongodb", "mysql_async", "nix", "ordered-float 5.1.0", @@ -919,6 +943,15 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -962,6 +995,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -1115,6 +1154,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + [[package]] name = "deranged" version = "0.5.8" @@ -1125,6 +1170,28 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -1156,6 +1223,29 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -1244,6 +1334,18 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1651,6 +1753,52 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.9.2", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "moka", + "once_cell", + "parking_lot", + "rand 0.9.2", + "resolv-conf", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1894,7 +2042,7 @@ dependencies = [ "serde_with", "strum", "tokio", - "typed-builder", + "typed-builder 0.20.1", "url", "uuid", "zstd", @@ -1917,7 +2065,7 @@ dependencies = [ "serde_json", "tokio", "tracing", - "typed-builder", + "typed-builder 0.20.1", "uuid", ] @@ -2064,6 +2212,18 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2 0.5.10", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2368,6 +2528,54 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.117", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2446,6 +2654,82 @@ dependencies = [ "uuid", ] +[[package]] +name = "mongocrypt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.5+1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" + +[[package]] +name = "mongodb" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803dd859e8afa084c255a8effd8000ff86f7c8076a50cd6d8c99e8f3496f75c2" +dependencies = [ + "base64 0.22.1", + "bitflags", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-proto", + "hickory-resolver", + "hmac", + "macro_magic", + "md-5", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2", + "percent-encoding", + "rand 0.9.2", + "rustc_version_runtime", + "rustls", + "rustversion", + "serde", + "serde_bytes", + "serde_with", + "sha1", + "sha2", + "socket2 0.6.2", + "stringprep", + "strsim", + "take_mut", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "typed-builder 0.22.0", + "uuid", + "webpki-roots", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a973ef3dd3dbc6f6e65bbdecfd9ec5e781b9e7493b0f369a7c62e35d8e5ae2c8" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "murmur3" version = "0.5.2" @@ -2674,6 +2958,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2857,6 +3145,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", +] + [[package]] name = "pem" version = "3.0.6" @@ -3497,6 +3794,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "ring" version = "0.17.14" @@ -3616,6 +3919,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3859,6 +4172,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap 2.13.0", "itoa", "memchr", "serde", @@ -4167,6 +4481,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + [[package]] name = "tap" version = "1.0.1" @@ -4443,6 +4763,7 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -4648,7 +4969,16 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.20.1", +] + +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro 0.22.0", ] [[package]] @@ -4662,6 +4992,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "typenum" version = "1.19.0" @@ -4695,6 +5036,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -5013,6 +5360,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -5112,6 +5465,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5154,6 +5516,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -5193,6 +5570,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5211,6 +5594,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5229,6 +5618,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5259,6 +5654,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -5277,6 +5678,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -5295,6 +5702,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -5313,6 +5726,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -5334,6 +5753,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/Cargo.toml b/Cargo.toml index 0888a08..dc7e4fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ description = "A Change Data Capture (CDC) pipeline in Rust" repository = "https://github.com/manfredcml/cdcflow" homepage = "https://github.com/manfredcml/cdcflow" readme = "README.md" -keywords = ["cdc", "change-data-capture", "replication", "postgres", "mysql"] +keywords = ["cdc", "change-data-capture", "replication", "postgres", "mongodb"] categories = ["database", "command-line-utilities"] exclude = ["benchmark/"] @@ -33,6 +33,7 @@ arrow-schema = "57" parquet = { version = "57", default-features = false, features = ["arrow"] } async-trait = "0.1" rdkafka = { version = "0.39.0", features = ["cmake-build", "tokio"] } +mongodb = "3" ordered-float = { version = "5.1.0", features = ["serde"] } chrono = "0.4" axum = { version = "0.8", features = ["json"] } @@ -50,4 +51,4 @@ approx_constant = "allow" tower = "0.5" tempfile = "3" testcontainers = "0.27.1" -testcontainers-modules = { version = "0.15.0", features = ["postgres", "mysql", "kafka"] } +testcontainers-modules = { version = "0.15.0", features = ["postgres", "mysql", "kafka", "mongo"] } diff --git a/README.md b/README.md index ebeb925..2100ddc 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ destinations. ## Features -- **Database sources**: PostgreSQL and MySQL +- **Database sources**: PostgreSQL, MySQL, and MongoDB - **Sink destinations**: Stdout, Kafka, PostgreSQL, Apache Iceberg - **Offset stores**: SQLite, Memoryfor resumable streaming after restarts - **Sync modes**: CDC (append-only changelog) and Replication (live replica of source tables) @@ -20,7 +20,7 @@ destinations. ``` ┌────────────┐ ┌────────────┐ ┌──────────────┐ │ Source │────▶│ Pipeline │────▶│ Sink │ -│ PG / MySQL │ │ (batch + │ │ Kafka / PG / │ +│PG/MySQL/Mon│ │ (batch + │ │ Kafka / PG / │ │ │ │ flush) │ │ Iceberg / Out│ └────────────┘ └─────┬──────┘ └──────────────┘ │ @@ -42,7 +42,7 @@ checkpoints progress in the offset store. On restart, the pipeline resumes from The standalone mode runs a single pipeline without the admin server. This is ideal for local testing and development. ```bash -# Start local infrastructure (PostgreSQL, MySQL, Kafka, MinIO, Iceberg, Trino) +# Start local infrastructure (PostgreSQL, MySQL, MongoDB, Kafka, MinIO, Iceberg, Trino) # You can optionally start just the components you need for testing a specific source/sink combo cd example && docker compose up -d && cd .. diff --git a/example/configs/mongo-to-iceberg.json b/example/configs/mongo-to-iceberg.json new file mode 100644 index 0000000..16d4b0b --- /dev/null +++ b/example/configs/mongo-to-iceberg.json @@ -0,0 +1,30 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "iceberg", + "catalog": { + "type": "rest", + "uri": "http://localhost:8181", + "warehouse": "s3://warehouse/iceberg", + "properties": { + "s3.endpoint": "http://localhost:9000", + "s3.access-key-id": "minioadmin", + "s3.secret-access-key": "minioadmin", + "s3.path-style-access": "true", + "s3.region": "us-east-1" + } + }, + "namespace": "demo", + "table_prefix": "" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-iceberg" + } +} diff --git a/example/configs/mongo-to-kafka.json b/example/configs/mongo-to-kafka.json new file mode 100644 index 0000000..a888fae --- /dev/null +++ b/example/configs/mongo-to-kafka.json @@ -0,0 +1,18 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "kafka", + "brokers": "localhost:9092", + "topic_prefix": "cdc" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-kafka" + } +} diff --git a/example/configs/mongo-to-pg.json b/example/configs/mongo-to-pg.json new file mode 100644 index 0000000..478b576 --- /dev/null +++ b/example/configs/mongo-to-pg.json @@ -0,0 +1,23 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "postgres", + "host": "localhost", + "port": 5433, + "user": "postgres", + "password": "postgres", + "database": "cdc_target", + "schema": "public", + "table_prefix": "" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-pg" + } +} diff --git a/example/configs/mongo-to-stdout.json b/example/configs/mongo-to-stdout.json new file mode 100644 index 0000000..cf7fb33 --- /dev/null +++ b/example/configs/mongo-to-stdout.json @@ -0,0 +1,16 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "stdout" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-stdout" + } +} diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 19b223a..e27eb42 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -50,6 +50,28 @@ services: timeout: 5s retries: 10 + mongodb: + image: mongo:7 + container_name: cdc-mongodb + ports: + - "27017:27017" + command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] + healthcheck: + test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 5s + retries: 10 + + mongodb-init: + image: mongo:7 + container_name: cdc-mongodb-init + depends_on: + mongodb: + condition: service_healthy + volumes: + - ./init/mongodb/init.js:/init.js + entrypoint: ["mongosh", "--host", "mongodb", "--file", "/init.js"] + # ────────────────────────────────────────────── # Infrastructure # ────────────────────────────────────────────── diff --git a/example/init/mongodb/init.js b/example/init/mongodb/init.js new file mode 100644 index 0000000..cb0eb35 --- /dev/null +++ b/example/init/mongodb/init.js @@ -0,0 +1,41 @@ +// Initialize replica set (required for change streams) +rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "mongodb:27017" }] }); + +// Wait for primary election +let attempts = 0; +while (!rs.isMaster().ismaster && attempts < 30) { + sleep(1000); + attempts++; +} + +if (!rs.isMaster().ismaster) { + print("ERROR: Failed to elect primary after 30 seconds"); + quit(1); +} + +print("Replica set initialized, primary elected"); + +// Create demo database and collections +const db = db.getSiblingDB("demo"); + +// Enable pre/post images for change streams (MongoDB 6.0+) +db.createCollection("users", { + changeStreamPreAndPostImages: { enabled: true }, +}); +db.createCollection("orders", { + changeStreamPreAndPostImages: { enabled: true }, +}); + +// Insert sample data +db.users.insertMany([ + { _id: ObjectId(), name: "Alice", email: "alice@example.com", age: 30, active: true }, + { _id: ObjectId(), name: "Bob", email: "bob@example.com", age: 25, active: true }, + { _id: ObjectId(), name: "Charlie", email: "charlie@example.com", age: 35, active: false }, +]); + +db.orders.insertMany([ + { _id: ObjectId(), user: "Alice", product: "Widget", quantity: 2, price: 19.99 }, + { _id: ObjectId(), user: "Bob", product: "Gadget", quantity: 1, price: 49.99 }, +]); + +print("Demo data inserted: " + db.users.countDocuments() + " users, " + db.orders.countDocuments() + " orders"); diff --git a/src/config/mod.rs b/src/config/mod.rs index f532d47..0dfb246 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,5 +1,6 @@ mod iceberg_sink; mod kafka_sink; +mod mongodb_source; mod mysql_source; mod offset; mod postgres_sink; @@ -9,6 +10,7 @@ mod source; pub use self::iceberg_sink::{IcebergCatalogConfig, IcebergSinkConfig, RestCatalogConfig}; pub use self::kafka_sink::KafkaSinkConfig; +pub use self::mongodb_source::MongodbSourceConfig; pub use self::mysql_source::MySqlSourceConfig; pub use self::offset::OffsetConfig; pub use self::postgres_sink::PostgresSinkConfig; @@ -37,6 +39,8 @@ pub enum SourceConnectionConfig { Postgres { url: String }, #[serde(rename = "mysql")] Mysql { url: String }, + #[serde(rename = "mongodb")] + Mongodb { url: String }, } /// Top-level configuration for the CDC agent. diff --git a/src/config/mongodb_source.rs b/src/config/mongodb_source.rs new file mode 100644 index 0000000..7e50ab3 --- /dev/null +++ b/src/config/mongodb_source.rs @@ -0,0 +1,66 @@ +use serde::{Deserialize, Serialize}; + +/// MongoDB-specific source configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MongodbSourceConfig { + /// MongoDB connection string (e.g. `mongodb://host:port/?replicaSet=rs0`). + pub connection_string: String, + /// Database to capture changes from. + pub database: String, + /// Collections to capture. Empty means all collections. + #[serde(default)] + pub collections: Vec, +} + +#[cfg(test)] +mod tests { + use crate::config::{Config, SourceConfig}; + + #[test] + fn test_mongodb_config_deserialization() { + let json = r#"{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "mydb", + "collections": ["users", "orders"] + }, + "sink": { "type": "stdout" }, + "offset": { "type": "memory" } + }"#; + + let config: Config = serde_json::from_str(json).unwrap(); + match &config.source { + SourceConfig::Mongodb(mg) => { + assert_eq!( + mg.connection_string, + "mongodb://localhost:27017/?replicaSet=rs0" + ); + assert_eq!(mg.database, "mydb"); + assert_eq!(mg.collections, vec!["users", "orders"]); + } + _ => panic!("expected Mongodb config"), + } + } + + #[test] + fn test_mongodb_config_empty_collections_default() { + let json = r#"{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "mydb" + }, + "sink": { "type": "stdout" }, + "offset": { "type": "sqlite", "path": "/tmp/offset.db", "key": "mongo-offset" } + }"#; + + let config: Config = serde_json::from_str(json).unwrap(); + match &config.source { + SourceConfig::Mongodb(mg) => { + assert!(mg.collections.is_empty()); + } + _ => panic!("expected Mongodb config"), + } + } +} diff --git a/src/config/source.rs b/src/config/source.rs index 9f6063c..d66ddc4 100644 --- a/src/config/source.rs +++ b/src/config/source.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; +use super::mongodb_source::MongodbSourceConfig; use super::mysql_source::MySqlSourceConfig; use super::postgres_source::PostgresSourceConfig; @@ -11,6 +12,8 @@ pub enum SourceConfig { Postgres(PostgresSourceConfig), #[serde(rename = "mysql")] Mysql(MySqlSourceConfig), + #[serde(rename = "mongodb")] + Mongodb(MongodbSourceConfig), } impl SourceConfig { @@ -47,6 +50,9 @@ impl SourceConfig { my.database, ), }, + SourceConfig::Mongodb(mg) => super::SourceConnectionConfig::Mongodb { + url: mg.connection_string.clone(), + }, } } } @@ -120,6 +126,21 @@ mod tests { } } + #[test] + fn test_mongodb_to_connection_config() { + let config = SourceConfig::Mongodb(super::MongodbSourceConfig { + connection_string: "mongodb://localhost:27017/?replicaSet=rs0".into(), + database: "mydb".into(), + collections: vec![], + }); + match config.to_connection_config() { + SourceConnectionConfig::Mongodb { url } => { + assert_eq!(url, "mongodb://localhost:27017/?replicaSet=rs0"); + } + _ => panic!("expected Mongodb variant"), + } + } + #[test] fn test_mysql_to_connection_config_without_password() { let config = SourceConfig::Mysql(MySqlSourceConfig { diff --git a/src/error.rs b/src/error.rs index 4f6ee5b..932fed9 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,9 @@ pub enum CdcError { #[error("mysql error: {0}")] Mysql(String), + #[error("mongodb error: {0}")] + Mongodb(String), + #[error("iceberg error: {0}")] Iceberg(String), diff --git a/src/lib.rs b/src/lib.rs index 5458342..6272e53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ //! # cdcflow //! //! A Change Data Capture (CDC) pipeline that captures row-level changes -//! from PostgreSQL and MySQL, streaming them to Kafka, PostgreSQL, +//! from PostgreSQL, MySQL, and MongoDB, streaming them to Kafka, PostgreSQL, //! Apache Iceberg, or stdout. //! //! The pipeline reads events from a [`source`], batches them by transaction diff --git a/src/main.rs b/src/main.rs index e89d869..b684dd9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ use cdcflow::sink::kafka::KafkaSink; use cdcflow::sink::postgres::PostgresSink; use cdcflow::sink::stdout::StdoutSink; use cdcflow::sink::Sink; +use cdcflow::source::mongodb::MongodbSource; use cdcflow::source::mysql::MySqlSource; use cdcflow::source::postgres::PostgresSource; use cdcflow::worker_http::registration::AdminClient; @@ -23,7 +24,7 @@ use cdcflow::worker_http::WorkerHttpServer; #[derive(Parser, Debug)] #[command( name = "cdcflow", - about = "Change Data Capture agent for PostgreSQL and MySQL" + about = "Change Data Capture agent for PostgreSQL, MySQL, and MongoDB" )] struct Cli { #[command(subcommand)] @@ -646,6 +647,26 @@ async fn run_with_sink( pipeline.run(shutdown).await } }, + SourceConfig::Mongodb(mg_config) => match offset_config { + OffsetConfig::Sqlite { path, key } => { + let os = SqliteOffsetStore::new(&path, &key)?; + let mut pipeline = + Pipeline::new(MongodbSource::new(mg_config, os.clone()), sink, os); + if let Some(m) = metrics { + pipeline = pipeline.with_metrics(m); + } + pipeline.run(shutdown).await + } + OffsetConfig::Memory => { + let os = MemoryOffsetStore::new(); + let mut pipeline = + Pipeline::new(MongodbSource::new(mg_config, os.clone()), sink, os); + if let Some(m) = metrics { + pipeline = pipeline.with_metrics(m); + } + pipeline.run(shutdown).await + } + }, } } diff --git a/src/schema/discovery.rs b/src/schema/discovery.rs index 35dbc11..3ec8c3e 100644 --- a/src/schema/discovery.rs +++ b/src/schema/discovery.rs @@ -18,6 +18,7 @@ pub async fn fetch_columns( fetch_columns_postgres(url, schema, table).await } SourceConnectionConfig::Mysql { url } => fetch_columns_mysql(url, schema, table).await, + SourceConnectionConfig::Mongodb { url } => fetch_columns_mongodb(url, schema, table).await, } } @@ -32,6 +33,7 @@ pub async fn fetch_primary_keys( fetch_primary_keys_postgres(url, schema, table).await } SourceConnectionConfig::Mysql { url } => fetch_primary_keys_mysql(url, schema, table).await, + SourceConnectionConfig::Mongodb { .. } => Ok(vec!["_id".to_string()]), } } @@ -52,6 +54,9 @@ pub async fn fetch_column_subset( SourceConnectionConfig::Mysql { url } => { fetch_column_subset_mysql(url, schema, table, column_names).await } + SourceConnectionConfig::Mongodb { url } => { + fetch_column_subset_mongodb(url, schema, table, column_names).await + } } } @@ -340,3 +345,86 @@ async fn fetch_column_subset_mysql( Ok(result) } + +/// Fetch columns by sampling documents from a MongoDB collection. +/// +/// Since MongoDB is schemaless, we sample up to 100 documents and build a +/// union of all top-level field names, inferring BSON type from the first +/// non-null value seen per field. +async fn fetch_columns_mongodb( + url: &str, + database: &str, + collection: &str, +) -> Result> { + use mongodb::bson::doc; + + let client = mongodb::Client::with_uri_str(url) + .await + .map_err(|e| CdcError::Schema(format!("mongodb connect: {e}")))?; + + let db = client.database(database); + let coll = db.collection::(collection); + + let mut cursor = coll + .find(doc! {}) + .limit(100) + .await + .map_err(|e| CdcError::Schema(format!("mongodb find: {e}")))?; + + let mut field_types: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + use futures_util::StreamExt; + while let Some(result) = cursor.next().await { + let doc = result.map_err(|e| CdcError::Schema(format!("mongodb cursor: {e}")))?; + for (key, value) in &doc { + field_types.entry(key.clone()).or_insert_with(|| { + crate::source::mongodb::bson_mapping::bson_type_string(value).to_string() + }); + } + } + + let dialect = SourceDialect::Mongodb; + let columns: Vec = field_types + .into_iter() + .enumerate() + .map(|(i, (name, type_str))| { + let canonical_type = parse_source_type(dialect, &type_str); + ColumnInfo { + name, + source_type: type_str, + canonical_type, + is_nullable: true, + ordinal_position: i as i32, + } + }) + .collect(); + + Ok(columns) +} + +async fn fetch_column_subset_mongodb( + url: &str, + database: &str, + collection: &str, + column_names: &[String], +) -> Result> { + let all_columns = fetch_columns_mongodb(url, database, collection).await?; + let type_map: std::collections::HashMap = all_columns + .into_iter() + .map(|col| (col.name.clone(), col)) + .collect(); + + Ok(column_names + .iter() + .map(|name| { + type_map.get(name).cloned().unwrap_or_else(|| ColumnInfo { + name: name.clone(), + source_type: "string".into(), + canonical_type: super::CanonicalType::Text, + is_nullable: true, + ordinal_position: 0, + }) + }) + .collect()) +} diff --git a/src/schema/mod.rs b/src/schema/mod.rs index def633d..f1c4685 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -11,6 +11,7 @@ use crate::event::ColumnValue; pub enum SourceDialect { Postgres, Mysql, + Mongodb, } impl From<&SourceConnectionConfig> for SourceDialect { @@ -18,6 +19,7 @@ impl From<&SourceConnectionConfig> for SourceDialect { match conn { SourceConnectionConfig::Postgres { .. } => SourceDialect::Postgres, SourceConnectionConfig::Mysql { .. } => SourceDialect::Mysql, + SourceConnectionConfig::Mongodb { .. } => SourceDialect::Mongodb, } } } diff --git a/src/schema/type_mapping.rs b/src/schema/type_mapping.rs index 92affaa..b88a6ca 100644 --- a/src/schema/type_mapping.rs +++ b/src/schema/type_mapping.rs @@ -9,6 +9,7 @@ pub fn parse_source_type(dialect: SourceDialect, raw_type: &str) -> CanonicalTyp match dialect { SourceDialect::Postgres => parse_postgres_type(&dt), SourceDialect::Mysql => parse_mysql_type(&dt), + SourceDialect::Mongodb => parse_mongodb_type(&dt), } } @@ -143,6 +144,21 @@ fn parse_mysql_type(dt: &str) -> CanonicalType { } } +fn parse_mongodb_type(dt: &str) -> CanonicalType { + match dt { + "objectid" | "string" => CanonicalType::Text, + "int" => CanonicalType::Int, + "long" => CanonicalType::BigInt, + "double" => CanonicalType::Double, + "bool" => CanonicalType::Boolean, + "date" => CanonicalType::TimestampTz, + "bindata" => CanonicalType::Binary, + "object" | "array" => CanonicalType::Json, + "decimal" => CanonicalType::Text, // Decimal128 has 34 digits precision, store as text + _ => CanonicalType::Text, + } +} + #[cfg(test)] mod tests { use super::*; @@ -503,6 +519,47 @@ mod tests { ); } + #[test] + fn test_mongodb_type_mapping() { + let cases = vec![ + ("objectid", CanonicalType::Text), + ("string", CanonicalType::Text), + ("int", CanonicalType::Int), + ("long", CanonicalType::BigInt), + ("double", CanonicalType::Double), + ("bool", CanonicalType::Boolean), + ("date", CanonicalType::TimestampTz), + ("bindata", CanonicalType::Binary), + ("object", CanonicalType::Json), + ("array", CanonicalType::Json), + ("decimal", CanonicalType::Text), + ("unknown_bson_type", CanonicalType::Text), + ]; + for (input, expected) in cases { + assert_eq!( + parse_source_type(SourceDialect::Mongodb, input), + expected, + "failed for MongoDB type '{input}'" + ); + } + } + + #[test] + fn test_mongodb_case_insensitive() { + assert_eq!( + parse_source_type(SourceDialect::Mongodb, "ObjectId"), + CanonicalType::Text + ); + assert_eq!( + parse_source_type(SourceDialect::Mongodb, "STRING"), + CanonicalType::Text + ); + assert_eq!( + parse_source_type(SourceDialect::Mongodb, "Int"), + CanonicalType::Int + ); + } + #[test] fn test_mysql_case_insensitive() { assert_eq!( diff --git a/src/sink/postgres/mod.rs b/src/sink/postgres/mod.rs index b785387..ca1f1e3 100644 --- a/src/sink/postgres/mod.rs +++ b/src/sink/postgres/mod.rs @@ -135,7 +135,9 @@ impl PostgresSink { .map(|col| { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; (col.name.clone(), sql_type) }) @@ -229,7 +231,9 @@ impl PostgresSink { .map(|col| { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; (col.name.clone(), sql_type) }) @@ -306,7 +310,9 @@ impl PostgresSink { for col in &source_columns { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; typed_cols.push((col.name.clone(), sql_type.clone())); typed_cols.push((format!("_old_{}", col.name), sql_type)); @@ -320,7 +326,9 @@ impl PostgresSink { for col in &source_columns { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; info_mut.columns.push(col.name.clone()); info_mut.column_types.insert(col.name.clone(), sql_type); @@ -425,7 +433,9 @@ impl PostgresSink { .map(|col| { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; (col.name.clone(), sql_type) }) diff --git a/src/source/mod.rs b/src/source/mod.rs index bd46693..239f61a 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -1,3 +1,4 @@ +pub mod mongodb; pub mod mysql; pub mod postgres; diff --git a/src/source/mongodb/bson_mapping.rs b/src/source/mongodb/bson_mapping.rs new file mode 100644 index 0000000..2780294 --- /dev/null +++ b/src/source/mongodb/bson_mapping.rs @@ -0,0 +1,271 @@ +use std::collections::BTreeMap; + +use mongodb::bson::{Bson, Document}; + +use crate::event::{ColumnValue, Row}; + +/// Convert a BSON value to a `ColumnValue`. +/// +/// Top-level fields are flattened: nested documents and arrays are serialized +/// as JSON text rather than recursed into. +pub fn bson_to_column_value(bson: &Bson) -> ColumnValue { + match bson { + Bson::Null => ColumnValue::Null, + Bson::Boolean(b) => ColumnValue::Bool(*b), + Bson::Int32(i) => ColumnValue::Int(*i as i64), + Bson::Int64(i) => ColumnValue::Int(*i), + Bson::Double(d) => ColumnValue::float(*d), + Bson::String(s) => ColumnValue::Text(s.clone()), + Bson::ObjectId(oid) => ColumnValue::Text(oid.to_hex()), + Bson::DateTime(dt) => { + // Convert millis to ISO 8601 via chrono + let millis = dt.timestamp_millis(); + let secs = millis / 1000; + let nsecs = ((millis % 1000) * 1_000_000) as u32; + let ts = chrono::DateTime::from_timestamp(secs, nsecs) + .unwrap_or_default() + .to_rfc3339(); + ColumnValue::Timestamp(ts) + } + Bson::Binary(bin) => ColumnValue::Bytes(bin.bytes.clone()), + Bson::Decimal128(d) => ColumnValue::Text(d.to_string()), + Bson::Timestamp(ts) => { + // BSON Timestamp is (seconds, increment) — used internally by oplog. + // Format as seconds since epoch. + ColumnValue::Text(format!("{}:{}", ts.time, ts.increment)) + } + Bson::RegularExpression(re) => ColumnValue::Text(format!("/{}/{}", re.pattern, re.options)), + // Nested documents and arrays → JSON text + Bson::Document(_) | Bson::Array(_) => { + let json = serde_json::to_string(bson).unwrap_or_default(); + ColumnValue::Text(json) + } + // Remaining BSON types → text representation + _ => { + let json = serde_json::to_string(bson).unwrap_or_default(); + ColumnValue::Text(json) + } + } +} + +/// Convert a BSON document to a CDC `Row` (BTreeMap of field → ColumnValue). +pub fn document_to_row(doc: &Document) -> Row { + let mut row: BTreeMap = BTreeMap::new(); + for (key, value) in doc { + row.insert(key.clone(), bson_to_column_value(value)); + } + row +} + +/// Return the BSON type name string for schema inference. +pub fn bson_type_string(bson: &Bson) -> &'static str { + match bson { + Bson::Null => "null", + Bson::Boolean(_) => "bool", + Bson::Int32(_) => "int", + Bson::Int64(_) => "long", + Bson::Double(_) => "double", + Bson::String(_) => "string", + Bson::ObjectId(_) => "objectid", + Bson::DateTime(_) => "date", + Bson::Binary(_) => "bindata", + Bson::Decimal128(_) => "decimal", + Bson::Document(_) => "object", + Bson::Array(_) => "array", + Bson::Timestamp(_) => "string", + Bson::RegularExpression(_) => "string", + _ => "string", + } +} + +/// Infer schema from a set of documents by building a union of all top-level +/// fields. The type for each field is taken from the first non-null value seen. +pub fn infer_schema_from_documents(docs: &[Document]) -> Vec<(String, String)> { + let mut field_types: BTreeMap = BTreeMap::new(); + for doc in docs { + for (key, value) in doc { + field_types + .entry(key.clone()) + .or_insert_with(|| bson_type_string(value).to_string()); + } + } + field_types.into_iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use mongodb::bson::{doc, oid::ObjectId, Binary, Decimal128, Regex}; + + #[test] + fn test_bson_to_column_value_primitives() { + let cases: Vec<(Bson, ColumnValue)> = vec![ + (Bson::Null, ColumnValue::Null), + (Bson::Boolean(true), ColumnValue::Bool(true)), + (Bson::Boolean(false), ColumnValue::Bool(false)), + (Bson::Int32(42), ColumnValue::Int(42)), + (Bson::Int32(-1), ColumnValue::Int(-1)), + (Bson::Int64(i64::MAX), ColumnValue::Int(i64::MAX)), + (Bson::Double(3.14), ColumnValue::float(3.14)), + ( + Bson::String("hello".into()), + ColumnValue::Text("hello".into()), + ), + ]; + for (bson, expected) in cases { + assert_eq!(bson_to_column_value(&bson), expected, "for {bson:?}"); + } + } + + #[test] + fn test_bson_to_column_value_objectid() { + let oid = ObjectId::parse_str("507f1f77bcf86cd799439011").unwrap(); + let result = bson_to_column_value(&Bson::ObjectId(oid)); + assert_eq!(result, ColumnValue::Text("507f1f77bcf86cd799439011".into())); + } + + #[test] + fn test_bson_to_column_value_binary() { + let bin = Binary { + subtype: mongodb::bson::spec::BinarySubtype::Generic, + bytes: vec![0xDE, 0xAD], + }; + let result = bson_to_column_value(&Bson::Binary(bin)); + assert_eq!(result, ColumnValue::Bytes(vec![0xDE, 0xAD])); + } + + #[test] + fn test_bson_to_column_value_datetime() { + let dt = mongodb::bson::DateTime::from_millis(1700000000000); + let result = bson_to_column_value(&Bson::DateTime(dt)); + match result { + ColumnValue::Timestamp(s) => { + assert!(s.contains("2023"), "expected 2023 in timestamp: {s}"); + } + other => panic!("expected Timestamp, got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_decimal128() { + let d = Decimal128::from_bytes([0; 16]); + let result = bson_to_column_value(&Bson::Decimal128(d)); + match result { + ColumnValue::Text(_) => {} // Decimal128 stored as text + other => panic!("expected Text, got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_nested_doc() { + let bson = Bson::Document(doc! { "nested": "value" }); + let result = bson_to_column_value(&bson); + match result { + ColumnValue::Text(json) => { + assert!(json.contains("nested"), "expected JSON: {json}"); + } + other => panic!("expected Text (JSON), got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_array() { + let bson = Bson::Array(vec![Bson::Int32(1), Bson::Int32(2)]); + let result = bson_to_column_value(&bson); + match result { + ColumnValue::Text(json) => { + assert_eq!(json, "[1,2]"); + } + other => panic!("expected Text (JSON array), got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_regex() { + let re = Regex { + pattern: "^abc".into(), + options: "i".into(), + }; + let result = bson_to_column_value(&Bson::RegularExpression(re)); + assert_eq!(result, ColumnValue::Text("/^abc/i".into())); + } + + #[test] + fn test_document_to_row() { + let doc = doc! { + "_id": ObjectId::new(), + "name": "Alice", + "age": 30, + "active": true, + "score": 95.5, + }; + let row = document_to_row(&doc); + assert_eq!(row.len(), 5); + assert!(matches!(row.get("name"), Some(ColumnValue::Text(s)) if s == "Alice")); + assert_eq!(row.get("age"), Some(&ColumnValue::Int(30))); + assert_eq!(row.get("active"), Some(&ColumnValue::Bool(true))); + assert_eq!(row.get("score"), Some(&ColumnValue::float(95.5))); + } + + #[test] + fn test_document_to_row_empty() { + let doc = doc! {}; + let row = document_to_row(&doc); + assert!(row.is_empty()); + } + + #[test] + fn test_bson_type_string() { + let cases: Vec<(Bson, &str)> = vec![ + (Bson::Null, "null"), + (Bson::Boolean(true), "bool"), + (Bson::Int32(0), "int"), + (Bson::Int64(0), "long"), + (Bson::Double(0.0), "double"), + (Bson::String("".into()), "string"), + (Bson::ObjectId(ObjectId::new()), "objectid"), + (Bson::DateTime(mongodb::bson::DateTime::now()), "date"), + ( + Bson::Binary(Binary { + subtype: mongodb::bson::spec::BinarySubtype::Generic, + bytes: vec![], + }), + "bindata", + ), + (Bson::Document(doc! {}), "object"), + (Bson::Array(vec![]), "array"), + (Bson::Decimal128(Decimal128::from_bytes([0; 16])), "decimal"), + ]; + for (bson, expected) in cases { + assert_eq!(bson_type_string(&bson), expected, "for {bson:?}"); + } + } + + #[test] + fn test_infer_schema_from_documents() { + let docs = vec![ + doc! { "_id": ObjectId::new(), "name": "Alice", "age": 30 }, + doc! { "_id": ObjectId::new(), "name": "Bob", "email": "bob@example.com" }, + ]; + let schema = infer_schema_from_documents(&docs); + let names: Vec<&str> = schema.iter().map(|(n, _)| n.as_str()).collect(); + // BTreeMap gives sorted order + assert_eq!(names, vec!["_id", "age", "email", "name"]); + + // Types should be inferred from first seen + let type_map: BTreeMap<&str, &str> = schema + .iter() + .map(|(n, t)| (n.as_str(), t.as_str())) + .collect(); + assert_eq!(type_map["_id"], "objectid"); + assert_eq!(type_map["name"], "string"); + assert_eq!(type_map["age"], "int"); + assert_eq!(type_map["email"], "string"); + } + + #[test] + fn test_infer_schema_empty() { + let schema = infer_schema_from_documents(&[]); + assert!(schema.is_empty()); + } +} diff --git a/src/source/mongodb/change_stream.rs b/src/source/mongodb/change_stream.rs new file mode 100644 index 0000000..835d453 --- /dev/null +++ b/src/source/mongodb/change_stream.rs @@ -0,0 +1,110 @@ +use futures_util::StreamExt; +use mongodb::bson::{doc, Document, Timestamp}; +use mongodb::change_stream::event::ResumeToken; +use mongodb::options::FullDocumentBeforeChangeType; +use mongodb::options::FullDocumentType; +use mongodb::Client; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use crate::error::{CdcError, Result}; +use crate::source::SourceEvent; + +use super::event_converter::{change_event_to_cdc_event, serialize_resume_token, ConvertResult}; + +/// Determines where to start the change stream from. +pub enum ResumePoint { + /// Start after a snapshot at the given cluster time. + AfterSnapshot { snapshot_time: Timestamp }, + /// Resume from a previously saved resume token. + Token(ResumeToken), +} + +/// Run the change stream loop, sending CDC events to the pipeline. +/// +/// Supports filtering to specific collections via a `$match` pipeline stage. +/// Uses `fullDocument: "updateLookup"` to get the full document on updates, +/// and `fullDocumentBeforeChange: "whenAvailable"` to capture pre-images. +pub async fn run_change_stream( + client: &Client, + database: &str, + collections: &[String], + resume_point: ResumePoint, + sender: &mpsc::Sender, + shutdown: &CancellationToken, +) -> Result<()> { + let db = client.database(database); + + // Build pipeline to filter collections if specified + let pipeline: Vec = if collections.is_empty() { + vec![] + } else { + vec![doc! { + "$match": { + "ns.coll": { "$in": collections } + } + }] + }; + + let mut builder = db + .watch() + .pipeline(pipeline) + .full_document(FullDocumentType::UpdateLookup) + .full_document_before_change(FullDocumentBeforeChangeType::WhenAvailable); + + match &resume_point { + ResumePoint::AfterSnapshot { snapshot_time } => { + builder = builder.start_at_operation_time(*snapshot_time); + } + ResumePoint::Token(token) => { + builder = builder.resume_after(token.clone()); + } + } + + let mut stream = builder + .await + .map_err(|e| CdcError::Mongodb(format!("failed to open change stream: {e}")))?; + + tracing::info!(database, "change stream started"); + + loop { + tokio::select! { + _ = shutdown.cancelled() => { + tracing::info!("shutdown received, stopping change stream"); + return Ok(()); + } + next = stream.next() => { + match next { + Some(Ok(event)) => { + match change_event_to_cdc_event(&event, database)? { + ConvertResult::Event(cdc_event) => { + sender + .send(SourceEvent::Change(cdc_event)) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to send change event: {e}")))?; + + // Send commit with resume token as offset + let offset = serialize_resume_token(&event.id)?; + sender + .send(SourceEvent::Commit { offset }) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to send commit: {e}")))?; + } + ConvertResult::Skip => { + tracing::debug!(op_type = ?event.operation_type, "skipping unsupported change event"); + } + ConvertResult::Commit { .. } => {} + } + } + Some(Err(e)) => { + return Err(CdcError::Mongodb(format!("change stream error: {e}"))); + } + None => { + tracing::warn!("change stream ended unexpectedly"); + return Ok(()); + } + } + } + } + } +} diff --git a/src/source/mongodb/event_converter.rs b/src/source/mongodb/event_converter.rs new file mode 100644 index 0000000..6840b15 --- /dev/null +++ b/src/source/mongodb/event_converter.rs @@ -0,0 +1,128 @@ +use mongodb::bson::Document; +use mongodb::change_stream::event::{ChangeStreamEvent, ResumeToken}; + +use crate::error::{CdcError, Result}; +use crate::event::{CdcEvent, ChangeOp, Lsn, TableId}; + +use super::bson_mapping::document_to_row; + +/// Result of converting a change stream event. +pub enum ConvertResult { + /// A CDC event to send downstream. + Event(CdcEvent), + /// A commit with serialized resume token as offset. + Commit { offset: String }, + /// Skip this event (unsupported operation type). + Skip, +} + +/// Convert a MongoDB change stream event to a `CdcEvent`. +/// +/// Returns `ConvertResult::Event` for insert/update/replace/delete, +/// `ConvertResult::Skip` for unsupported operations (drop, rename, etc.). +pub fn change_event_to_cdc_event( + event: &ChangeStreamEvent, + database: &str, +) -> Result { + let op = match event.operation_type { + mongodb::change_stream::event::OperationType::Insert => ChangeOp::Insert, + mongodb::change_stream::event::OperationType::Update + | mongodb::change_stream::event::OperationType::Replace => ChangeOp::Update, + mongodb::change_stream::event::OperationType::Delete => ChangeOp::Delete, + _ => return Ok(ConvertResult::Skip), + }; + + let collection = event + .ns + .as_ref() + .and_then(|ns| ns.coll.as_deref()) + .unwrap_or("unknown"); + + let timestamp_us = event + .cluster_time + .map(|ts| ts.time as i64 * 1_000_000) + .or_else(|| event.wall_time.map(|wt| wt.timestamp_millis() * 1000)) + .unwrap_or(0); + + let new = event.full_document.as_ref().map(document_to_row); + let old = event + .full_document_before_change + .as_ref() + .map(document_to_row); + + let cdc_event = CdcEvent { + lsn: Lsn(0), + timestamp_us, + xid: 0, + table: TableId { + schema: database.to_string(), + name: collection.to_string(), + oid: 0, + }, + op, + new, + old, + primary_key_columns: vec!["_id".to_string()], + }; + + Ok(ConvertResult::Event(cdc_event)) +} + +/// Convert a snapshot document to a `CdcEvent`. +pub fn snapshot_doc_to_cdc_event(doc: &Document, database: &str, collection: &str) -> CdcEvent { + let row = document_to_row(doc); + CdcEvent { + lsn: Lsn(0), + timestamp_us: chrono::Utc::now().timestamp_micros(), + xid: 0, + table: TableId { + schema: database.to_string(), + name: collection.to_string(), + oid: 0, + }, + op: ChangeOp::Snapshot, + new: Some(row), + old: None, + primary_key_columns: vec!["_id".to_string()], + } +} + +/// Serialize a resume token to a JSON string for offset storage. +pub fn serialize_resume_token(token: &ResumeToken) -> Result { + serde_json::to_string(token) + .map_err(|e| CdcError::Mongodb(format!("failed to serialize resume token: {e}"))) +} + +/// Deserialize a resume token from a JSON string. +pub fn deserialize_resume_token(s: &str) -> Result { + serde_json::from_str(s) + .map_err(|e| CdcError::Mongodb(format!("failed to deserialize resume token: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use mongodb::bson::doc; + + #[test] + fn test_snapshot_doc_to_cdc_event() { + let doc = doc! { + "_id": "abc123", + "name": "Alice", + "age": 30, + }; + let event = snapshot_doc_to_cdc_event(&doc, "mydb", "users"); + assert_eq!(event.op, ChangeOp::Snapshot); + assert_eq!(event.table.schema, "mydb"); + assert_eq!(event.table.name, "users"); + assert_eq!(event.primary_key_columns, vec!["_id"]); + assert!(event.new.is_some()); + assert!(event.old.is_none()); + + let row = event.new.unwrap(); + assert!( + matches!(row.get("name"), Some(crate::event::ColumnValue::Text(s)) if s == "Alice") + ); + assert_eq!(row.get("age"), Some(&crate::event::ColumnValue::Int(30))); + } +} diff --git a/src/source/mongodb/mod.rs b/src/source/mongodb/mod.rs new file mode 100644 index 0000000..467e7e5 --- /dev/null +++ b/src/source/mongodb/mod.rs @@ -0,0 +1,95 @@ +pub mod bson_mapping; +pub mod change_stream; +pub mod event_converter; +pub mod snapshot; + +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use crate::config::MongodbSourceConfig; +use crate::error::{CdcError, Result}; +use crate::offset::OffsetStore; +use crate::source::Source; + +use self::change_stream::{run_change_stream, ResumePoint}; +use self::event_converter::deserialize_resume_token; +use self::snapshot::perform_snapshot; + +/// MongoDB CDC source implementing the Source trait. +/// +/// Orchestrates: check offset → snapshot if needed → stream change events. +/// Uses the official `mongodb` crate driver. Requires a replica set deployment. +pub struct MongodbSource { + config: MongodbSourceConfig, + offset_store: O, +} + +impl MongodbSource { + pub fn new(config: MongodbSourceConfig, offset_store: O) -> Self { + Self { + config, + offset_store, + } + } +} + +impl Source for MongodbSource { + async fn start( + &mut self, + sender: mpsc::Sender, + shutdown: CancellationToken, + ) -> Result<()> { + let client = mongodb::Client::with_uri_str(&self.config.connection_string) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to connect: {e}")))?; + + // Phase 1: Determine start position (snapshot or resume) + let saved_offset = self.offset_store.load().await?; + + let resume_point = match saved_offset { + Some(offset_str) => { + tracing::info!("resuming from saved offset, skipping snapshot"); + let token = deserialize_resume_token(&offset_str)?; + ResumePoint::Token(token) + } + None => { + tracing::info!("no saved offset, performing initial snapshot"); + let result = perform_snapshot( + &client, + &self.config.database, + &self.config.collections, + &sender, + ) + .await?; + + if shutdown.is_cancelled() { + return Ok(()); + } + + ResumePoint::AfterSnapshot { + snapshot_time: result.snapshot_time, + } + } + }; + + if shutdown.is_cancelled() { + return Ok(()); + } + + // Phase 2: Stream change events + tracing::info!( + database = %self.config.database, + "starting change stream" + ); + + run_change_stream( + &client, + &self.config.database, + &self.config.collections, + resume_point, + &sender, + &shutdown, + ) + .await + } +} diff --git a/src/source/mongodb/snapshot.rs b/src/source/mongodb/snapshot.rs new file mode 100644 index 0000000..0079e22 --- /dev/null +++ b/src/source/mongodb/snapshot.rs @@ -0,0 +1,107 @@ +use futures_util::StreamExt; +use mongodb::bson::{doc, Timestamp}; +use mongodb::Client; +use tokio::sync::mpsc; + +use crate::error::{CdcError, Result}; +use crate::source::SourceEvent; + +use super::event_converter::snapshot_doc_to_cdc_event; + +/// Result of a snapshot operation. +pub struct SnapshotResult { + /// The cluster time recorded before the snapshot started. + /// Used to start change stream from the correct point. + pub snapshot_time: Timestamp, +} + +/// Perform initial snapshot of the specified collections. +/// +/// 1. Record the current cluster time via a `ping` command. +/// 2. Scan each collection with `find({})`. +/// 3. Send Change + Checkpoint events for each document batch. +pub async fn perform_snapshot( + client: &Client, + database: &str, + collections: &[String], + sender: &mpsc::Sender, +) -> Result { + let db = client.database(database); + + // Record cluster time before snapshot + let ping_result = db + .run_command(doc! { "ping": 1 }) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to ping for cluster time: {e}")))?; + + let snapshot_time = ping_result + .get("operationTime") + .and_then(|v| v.as_timestamp()) + .ok_or_else(|| { + CdcError::Mongodb("ping response missing operationTime — is this a replica set?".into()) + })?; + + tracing::info!( + snapshot_time = %format!("{}:{}", snapshot_time.time, snapshot_time.increment), + "recorded cluster time for snapshot" + ); + + // Determine collections to snapshot + let collection_names = if collections.is_empty() { + db.list_collection_names() + .await + .map_err(|e| CdcError::Mongodb(format!("failed to list collections: {e}")))? + } else { + collections.to_vec() + }; + + for coll_name in &collection_names { + tracing::info!(collection = %coll_name, "starting snapshot"); + let coll = db.collection::(coll_name); + + let mut cursor = coll + .find(doc! {}) + .await + .map_err(|e| CdcError::Mongodb(format!("snapshot find on {coll_name}: {e}")))?; + + let mut count: u64 = 0; + while let Some(result) = cursor.next().await { + let doc = result + .map_err(|e| CdcError::Mongodb(format!("snapshot cursor on {coll_name}: {e}")))?; + + let event = snapshot_doc_to_cdc_event(&doc, database, coll_name); + sender + .send(SourceEvent::Change(event)) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to send snapshot event: {e}")))?; + + count += 1; + + // Checkpoint every 1000 documents + if count.is_multiple_of(1000) { + let offset = format!( + "{{\"snapshot\":\"{}\",\"collection\":\"{}\",\"count\":{}}}", + coll_name, coll_name, count + ); + sender + .send(SourceEvent::Checkpoint { offset }) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to send checkpoint: {e}")))?; + } + } + + // Final checkpoint for this collection + let offset = format!( + "{{\"snapshot_complete\":\"{}\",\"count\":{}}}", + coll_name, count + ); + sender + .send(SourceEvent::Checkpoint { offset }) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to send checkpoint: {e}")))?; + + tracing::info!(collection = %coll_name, count, "snapshot complete"); + } + + Ok(SnapshotResult { snapshot_time }) +} diff --git a/tests/mongodb_integration.rs b/tests/mongodb_integration.rs new file mode 100644 index 0000000..e0b9ef7 --- /dev/null +++ b/tests/mongodb_integration.rs @@ -0,0 +1,538 @@ +//! Integration tests for the full CDC pipeline with MongoDB. +//! +//! These tests require Docker to be running. Run with: +//! cargo test -- --ignored +//! +//! Each test spins up a MongoDB container configured as a single-node replica set. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use mongodb::bson::doc; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, ImageExt}; +use testcontainers_modules::mongo::Mongo; +use tokio::time::timeout; +use tokio_util::sync::CancellationToken; + +use cdcflow::config::MongodbSourceConfig; +use cdcflow::error::Result; +use cdcflow::event::{CdcEvent, ChangeOp, ColumnValue}; +use cdcflow::offset::memory::MemoryOffsetStore; +use cdcflow::offset::OffsetStore; +use cdcflow::pipeline::Pipeline; +use cdcflow::sink::Sink; +use cdcflow::source::mongodb::MongodbSource; + +/// A sink that collects events into a shared Vec for assertions. +struct CollectorSink { + events: Arc>>, +} + +impl CollectorSink { + fn new() -> (Self, Arc>>) { + let events = Arc::new(Mutex::new(Vec::new())); + ( + Self { + events: events.clone(), + }, + events, + ) + } +} + +impl Sink for CollectorSink { + async fn write_batch(&mut self, events: &[CdcEvent]) -> Result<()> { + for event in events { + let json = serde_json::to_string_pretty(event).unwrap(); + println!("\n--- CDC Event ---\n{json}"); + } + self.events.lock().unwrap().extend_from_slice(events); + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + Ok(()) + } +} + +/// Start a MongoDB container configured as a single-node replica set. +async fn start_mongodb() -> (ContainerAsync, String, u16) { + let container = Mongo::default() + .with_tag("7") + .with_cmd(vec![ + "mongod".to_string(), + "--replSet".to_string(), + "rs0".to_string(), + "--bind_ip_all".to_string(), + ]) + .start() + .await + .expect("failed to start mongodb container"); + + let host_port = container + .get_host_port_ipv4(27017) + .await + .expect("failed to get host port"); + + let host = container + .get_host() + .await + .expect("failed to get host") + .to_string(); + + // Initialize replica set — use 127.0.0.1:{mapped_port} as the member host + // so MongoDB recognizes itself. The container binds to 0.0.0.0 and the + // mapped port is forwarded from the host, making this work with directConnection. + let conn_str = format!("mongodb://{}:{}/?directConnection=true", host, host_port); + let client = mongodb::Client::with_uri_str(&conn_str) + .await + .expect("failed to connect to mongodb"); + + let admin_db = client.database("admin"); + + // Use replSetInitiate with "localhost:27017" — the address MongoDB sees + // internally. We connect from the host via the mapped port using + // directConnection=true, but the replica set member must use the address + // visible inside the container. + admin_db + .run_command(doc! { + "replSetInitiate": { + "_id": "rs0", + "members": [{ "_id": 0, "host": "localhost:27017" }] + } + }) + .await + .expect("failed to initiate replica set"); + + // Wait for primary election + for _ in 0..30 { + tokio::time::sleep(Duration::from_secs(1)).await; + if let Ok(status) = admin_db.run_command(doc! { "replSetGetStatus": 1 }).await { + if let Some(members) = status.get_array("members").ok() { + for member in members { + if let Some(doc) = member.as_document() { + if doc.get_str("stateStr").ok() == Some("PRIMARY") { + // Primary elected, ready to go + return (container, host.to_string(), host_port); + } + } + } + } + } + } + + panic!("MongoDB replica set did not elect a primary within 30 seconds"); +} + +/// Build a connection string for the test container. +fn connection_string(host: &str, port: u16) -> String { + format!( + "mongodb://{}:{}/?replicaSet=rs0&directConnection=true", + host, port + ) +} + +/// Get a mongodb::Client connected to the test container. +async fn get_client(host: &str, port: u16) -> mongodb::Client { + let conn = connection_string(host, port); + mongodb::Client::with_uri_str(&conn) + .await + .expect("failed to create mongodb client") +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_snapshot_and_stream() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + // Insert initial documents + let db = client.database("testdb"); + let coll = db.collection::("test_coll"); + + // Enable pre/post images + db.run_command(doc! { + "collMod": "test_coll", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .ok(); // May fail if collection doesn't exist yet, that's fine + + // Drop and recreate with pre/post images enabled + coll.drop().await.ok(); + db.create_collection("test_coll") + .await + .expect("failed to create collection"); + db.run_command(doc! { + "collMod": "test_coll", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .expect("failed to enable change stream images"); + + coll.insert_many(vec![ + doc! { "name": "Alice", "age": 30 }, + doc! { "name": "Bob", "age": 25 }, + ]) + .await + .expect("failed to insert test data"); + + // Set up pipeline + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "testdb".into(), + collections: vec!["test_coll".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait for snapshot to complete + let snapshot_received = timeout(Duration::from_secs(30), async { + loop { + { + let evts = events.lock().unwrap(); + let snapshot_count = evts.iter().filter(|e| e.op == ChangeOp::Snapshot).count(); + if snapshot_count >= 2 { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await; + assert!( + snapshot_received.is_ok(), + "timed out waiting for snapshot events" + ); + + // Verify snapshot events + { + let evts = events.lock().unwrap(); + let snapshots: Vec<&CdcEvent> = + evts.iter().filter(|e| e.op == ChangeOp::Snapshot).collect(); + assert_eq!(snapshots.len(), 2, "expected 2 snapshot events"); + for snap in &snapshots { + assert_eq!(snap.table.schema, "testdb"); + assert_eq!(snap.table.name, "test_coll"); + assert_eq!(snap.primary_key_columns, vec!["_id"]); + assert!(snap.new.is_some()); + } + } + + // Now insert a new document to test streaming + coll.insert_one(doc! { "name": "Charlie", "age": 35 }) + .await + .expect("failed to insert streaming doc"); + + // Wait for the "Charlie" insert event specifically + let insert_received = timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + let has_charlie = evts.iter().any(|e| { + e.op == ChangeOp::Insert + && e.new + .as_ref() + .and_then(|r| r.get("name")) + .map(|v| matches!(v, ColumnValue::Text(s) if s == "Charlie")) + .unwrap_or(false) + }); + if has_charlie { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await; + assert!( + insert_received.is_ok(), + "timed out waiting for Charlie insert event" + ); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_update_and_delete() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("testdb2"); + + // Create collection with pre/post images + db.create_collection("items") + .await + .expect("failed to create collection"); + db.run_command(doc! { + "collMod": "items", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .expect("failed to enable change stream images"); + + let coll = db.collection::("items"); + + // Insert initial doc + coll.insert_one(doc! { "_id": "item1", "name": "Widget", "price": 10.0 }) + .await + .expect("failed to insert"); + + // Set up pipeline + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "testdb2".into(), + collections: vec!["items".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait for snapshot + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Snapshot) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for snapshot"); + + // Update the document + coll.update_one(doc! { "_id": "item1" }, doc! { "$set": { "price": 15.0 } }) + .await + .expect("failed to update"); + + // Wait for update event + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Update) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for update event"); + + // Verify update event + { + let evts = events.lock().unwrap(); + let updates: Vec<&CdcEvent> = evts.iter().filter(|e| e.op == ChangeOp::Update).collect(); + assert!(!updates.is_empty(), "expected update event"); + let update = updates.last().unwrap(); + // With updateLookup, full_document should have the updated doc + let new_row = update.new.as_ref().expect("update should have new row"); + assert_eq!( + new_row.get("name"), + Some(&ColumnValue::Text("Widget".into())) + ); + } + + // Delete the document + coll.delete_one(doc! { "_id": "item1" }) + .await + .expect("failed to delete"); + + // Wait for delete event + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Delete) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for delete event"); + + // Verify delete event + { + let evts = events.lock().unwrap(); + let deletes: Vec<&CdcEvent> = evts.iter().filter(|e| e.op == ChangeOp::Delete).collect(); + assert!(!deletes.is_empty(), "expected delete event"); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_resume_from_offset() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("testdb3"); + db.create_collection("resume_coll") + .await + .expect("failed to create collection"); + db.run_command(doc! { + "collMod": "resume_coll", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .expect("failed to enable change stream images"); + + let coll = db.collection::("resume_coll"); + + // Insert initial doc + coll.insert_one(doc! { "name": "First" }) + .await + .expect("failed to insert"); + + let conn_str = connection_string(&host, port); + + // Run 1: Snapshot + stream, then stop + let offset_store = MemoryOffsetStore::new(); + { + let config = MongodbSourceConfig { + connection_string: conn_str.clone(), + database: "testdb3".into(), + collections: vec!["resume_coll".into()], + }; + + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store.clone()); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait for snapshot + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Snapshot) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for snapshot"); + + // Insert while streaming to generate an offset + coll.insert_one(doc! { "name": "During" }) + .await + .expect("failed to insert"); + + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Insert) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for insert"); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; + } + + // Verify offset was saved + let saved_offset = offset_store.load().await.expect("failed to load offset"); + assert!(saved_offset.is_some(), "offset should have been saved"); + + // Insert a doc while pipeline is down + coll.insert_one(doc! { "name": "WhileDown" }) + .await + .expect("failed to insert while down"); + + // Run 2: Resume from offset + { + let config = MongodbSourceConfig { + connection_string: conn_str, + database: "testdb3".into(), + collections: vec!["resume_coll".into()], + }; + + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Should receive the "WhileDown" insert without re-snapshotting + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + let insert_names: Vec = evts + .iter() + .filter(|e| e.op == ChangeOp::Insert) + .filter_map(|e| { + e.new + .as_ref() + .and_then(|r| r.get("name")) + .and_then(|v| match v { + ColumnValue::Text(s) => Some(s.clone()), + _ => None, + }) + }) + .collect(); + if insert_names.contains(&"WhileDown".to_string()) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for WhileDown insert after resume"); + + // Verify no snapshot events (we resumed, not re-snapshotted) + { + let evts = events.lock().unwrap(); + let snapshot_count = evts.iter().filter(|e| e.op == ChangeOp::Snapshot).count(); + assert_eq!( + snapshot_count, 0, + "should not have snapshot events after resume" + ); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; + } +} From c3cd9b60d57c268850661798e83aec27832ca7f3 Mon Sep 17 00:00:00 2001 From: Manfred Lee Date: Thu, 26 Mar 2026 00:23:12 -0700 Subject: [PATCH 2/6] Update MongoDB connection strings and enhance integration tests --- example/configs/mongo-to-iceberg.json | 2 +- example/configs/mongo-to-kafka.json | 2 +- example/configs/mongo-to-pg.json | 2 +- example/configs/mongo-to-stdout.json | 2 +- example/docker-compose.yml | 2 +- example/init/mongodb/init.js | 2 +- tests/iceberg_integration.rs | 85 +++++++++ tests/kafka_integration.rs | 99 +++++++++++ tests/postgres_sink_integration.rs | 238 ++++++++++++++++++++++++++ 9 files changed, 428 insertions(+), 6 deletions(-) diff --git a/example/configs/mongo-to-iceberg.json b/example/configs/mongo-to-iceberg.json index 16d4b0b..d4a41ad 100644 --- a/example/configs/mongo-to-iceberg.json +++ b/example/configs/mongo-to-iceberg.json @@ -1,7 +1,7 @@ { "source": { "type": "mongodb", - "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", "database": "demo", "collections": ["users", "orders"] }, diff --git a/example/configs/mongo-to-kafka.json b/example/configs/mongo-to-kafka.json index a888fae..c9211f4 100644 --- a/example/configs/mongo-to-kafka.json +++ b/example/configs/mongo-to-kafka.json @@ -1,7 +1,7 @@ { "source": { "type": "mongodb", - "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", "database": "demo", "collections": ["users", "orders"] }, diff --git a/example/configs/mongo-to-pg.json b/example/configs/mongo-to-pg.json index 478b576..72cbf46 100644 --- a/example/configs/mongo-to-pg.json +++ b/example/configs/mongo-to-pg.json @@ -1,7 +1,7 @@ { "source": { "type": "mongodb", - "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", "database": "demo", "collections": ["users", "orders"] }, diff --git a/example/configs/mongo-to-stdout.json b/example/configs/mongo-to-stdout.json index cf7fb33..433a4eb 100644 --- a/example/configs/mongo-to-stdout.json +++ b/example/configs/mongo-to-stdout.json @@ -1,7 +1,7 @@ { "source": { "type": "mongodb", - "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", "database": "demo", "collections": ["users", "orders"] }, diff --git a/example/docker-compose.yml b/example/docker-compose.yml index e27eb42..a6261ab 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -70,7 +70,7 @@ services: condition: service_healthy volumes: - ./init/mongodb/init.js:/init.js - entrypoint: ["mongosh", "--host", "mongodb", "--file", "/init.js"] + entrypoint: ["mongosh", "mongodb://mongodb:27017/?directConnection=true", "--file", "/init.js"] # ────────────────────────────────────────────── # Infrastructure diff --git a/example/init/mongodb/init.js b/example/init/mongodb/init.js index cb0eb35..dc67a4e 100644 --- a/example/init/mongodb/init.js +++ b/example/init/mongodb/init.js @@ -1,5 +1,5 @@ // Initialize replica set (required for change streams) -rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "mongodb:27017" }] }); +rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost:27017" }] }); // Wait for primary election let attempts = 0; diff --git a/tests/iceberg_integration.rs b/tests/iceberg_integration.rs index 217a320..e690a41 100644 --- a/tests/iceberg_integration.rs +++ b/tests/iceberg_integration.rs @@ -651,3 +651,88 @@ async fn test_iceberg_replication_multi_batch() { "expected at least 1 delete manifest, got {delete_count}" ); } + +// --------------------------------------------------------------------------- +// MongoDB Source → Iceberg Sink Tests +// --------------------------------------------------------------------------- + +/// Test that MongoDB-shaped CDC events (schemaless, _id PK, mixed types) write +/// to Iceberg correctly in CDC mode. Uses a MongoDB SourceConnectionConfig so +/// the type mapping goes through the canonical Mongodb dialect path. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_iceberg_cdc_mongodb_source() { + // We still need a Postgres for source schema inference when the sink + // auto-creates the table. Create a source table that mirrors what a + // MongoDB collection's inferred schema would look like. + let env = TestEnv::start(&[ + "CREATE TABLE public.mongo_users (_id TEXT, name TEXT, age INTEGER, active BOOLEAN)", + ]) + .await; + + let config = env.sink_config(); + let source_conn = env.source_connection(); + let mut sink = IcebergSink::new(config, SinkMode::Cdc, source_conn) + .await + .unwrap(); + + let table_id = make_table_id("public", "mongo_users"); + + // MongoDB-shaped events: _id as text, mixed types + let events = vec![ + CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: table_id.clone(), + op: ChangeOp::Snapshot, + new: Some(BTreeMap::from([ + ( + "_id".into(), + ColumnValue::Text("507f1f77bcf86cd799439011".into()), + ), + ("name".into(), ColumnValue::Text("Alice".into())), + ("age".into(), ColumnValue::Text("30".into())), + ("active".into(), ColumnValue::Text("true".into())), + ])), + old: None, + primary_key_columns: vec!["_id".into()], + }, + CdcEvent { + lsn: Lsn(200), + timestamp_us: 1_700_000_000_000_001, + xid: 0, + table: table_id.clone(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([ + ( + "_id".into(), + ColumnValue::Text("507f1f77bcf86cd799439012".into()), + ), + ("name".into(), ColumnValue::Text("Bob".into())), + ("age".into(), ColumnValue::Text("25".into())), + ("active".into(), ColumnValue::Text("false".into())), + ])), + old: None, + primary_key_columns: vec!["_id".into()], + }, + ]; + + sink.write_batch(&events).await.unwrap(); + sink.flush().await.unwrap(); + + // Verify table was created and has data + let catalog = build_test_catalog(&env).await; + let ns = NamespaceIdent::from_strs(["default"]).unwrap(); + let ident = TableIdent::new(ns, "cdc_test_mongo_users".to_string()); + let table = catalog.load_table(&ident).await.unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = snapshot + .load_manifest_list(table.file_io(), &table.metadata_ref()) + .await + .unwrap(); + assert!( + !manifest_list.entries().is_empty(), + "should have at least one manifest" + ); +} diff --git a/tests/kafka_integration.rs b/tests/kafka_integration.rs index 6746c99..287230b 100644 --- a/tests/kafka_integration.rs +++ b/tests/kafka_integration.rs @@ -42,6 +42,29 @@ async fn start_kafka() -> (ContainerAsync, String) { (container, bootstrap_servers) } +/// Create a MongoDB-shaped CDC event (schema=database, _id as PK, mixed types). +fn make_mongo_event(database: &str, collection: &str, id: &str) -> CdcEvent { + CdcEvent { + lsn: Lsn::ZERO, + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: database.into(), + name: collection.into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some(BTreeMap::from([ + ("_id".into(), ColumnValue::Text(id.into())), + ("name".into(), ColumnValue::Text("Alice".into())), + ("age".into(), ColumnValue::Int(30)), + ("active".into(), ColumnValue::Bool(true)), + ])), + old: None, + primary_key_columns: vec!["_id".into()], + } +} + fn make_test_event(schema: &str, table_name: &str, id: &str) -> CdcEvent { CdcEvent { lsn: Lsn::ZERO, @@ -168,3 +191,79 @@ async fn test_kafka_sink_message_key() { let key = msg.key_view::().unwrap().unwrap(); assert_eq!(key, "myschema.mytable"); } + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_kafka_sink_with_mongodb_events() { + let (_container, brokers) = start_kafka().await; + + let config = KafkaSinkConfig { + brokers: brokers.clone(), + topic_prefix: "mongo".into(), + properties: Default::default(), + }; + let mut sink = KafkaSink::new(config).unwrap(); + + // Write MongoDB-shaped events from two collections + let events = vec![ + make_mongo_event("demo", "users", "507f1f77bcf86cd799439011"), + make_mongo_event("demo", "users", "507f1f77bcf86cd799439012"), + make_mongo_event("demo", "orders", "607f1f77bcf86cd799439099"), + ]; + sink.write_batch(&events).await.unwrap(); + sink.flush().await.unwrap(); + + // Consume and verify topic routing: {prefix}.{database}.{collection} + let consumer: StreamConsumer = ClientConfig::new() + .set("bootstrap.servers", &brokers) + .set("group.id", "test-mongo-topics") + .set("auto.offset.reset", "earliest") + .create() + .unwrap(); + + consumer + .subscribe(&["mongo.demo.users", "mongo.demo.orders"]) + .unwrap(); + + let mut users_messages = Vec::new(); + let mut orders_messages = Vec::new(); + + let consume_result = timeout(Duration::from_secs(15), async { + loop { + if let Ok(msg) = tokio::time::timeout(Duration::from_secs(2), consumer.recv()).await { + let msg = msg.unwrap(); + let topic = msg.topic().to_string(); + let payload = msg.payload_view::().unwrap().unwrap().to_string(); + if topic == "mongo.demo.users" { + users_messages.push(payload); + } else if topic == "mongo.demo.orders" { + orders_messages.push(payload); + } + if users_messages.len() >= 2 && !orders_messages.is_empty() { + break; + } + } + } + }) + .await; + + assert!(consume_result.is_ok(), "failed to consume all messages"); + assert_eq!(users_messages.len(), 2); + assert_eq!(orders_messages.len(), 1); + + // Verify JSON structure contains MongoDB fields + // Kafka uses nested format: {metadata, new: {values, types}, old} + let user_json: serde_json::Value = serde_json::from_str(&users_messages[0]).unwrap(); + assert_eq!(user_json["metadata"]["table"], "users"); + assert_eq!(user_json["metadata"]["schema"], "demo"); + assert_eq!(user_json["metadata"]["op"], "I"); + // MongoDB _id should appear in new.values + assert!(user_json["new"]["values"]["_id"].is_string()); + assert_eq!(user_json["new"]["values"]["name"], "Alice"); + assert_eq!(user_json["new"]["values"]["age"], 30); + assert_eq!(user_json["new"]["values"]["active"], true); + // Types should reflect ColumnValue variants + assert_eq!(user_json["new"]["types"]["_id"], "Text"); + assert_eq!(user_json["new"]["types"]["age"], "Int"); + assert_eq!(user_json["new"]["types"]["active"], "Bool"); +} diff --git a/tests/postgres_sink_integration.rs b/tests/postgres_sink_integration.rs index bf62a10..a2b5de8 100644 --- a/tests/postgres_sink_integration.rs +++ b/tests/postgres_sink_integration.rs @@ -163,6 +163,34 @@ fn make_truncate_event(schema: &str, table: &str) -> CdcEvent { } } +/// Create a MongoDB-shaped insert event (schema=database, _id as PK, mixed types). +fn make_mongo_insert(database: &str, collection: &str, id: &str, name: &str) -> CdcEvent { + CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: database.into(), + name: collection.into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text(id.into())), + ("name".to_string(), ColumnValue::Text(name.into())), + ("age".to_string(), ColumnValue::Int(30)), + ("active".to_string(), ColumnValue::Bool(true)), + ("score".to_string(), ColumnValue::float(95.5)), + ] + .into_iter() + .collect(), + ), + old: None, + primary_key_columns: vec!["_id".into()], + } +} + // ───────────────────────────────────────────────── // CDC Mode Tests // ───────────────────────────────────────────────── @@ -890,3 +918,213 @@ async fn test_replication_mode_schema_evolution() { let phone2: Option<&str> = rows[1].get("phone"); assert_eq!(phone2, Some("555-1234")); } + +// ───────────────────────────────────────────────── +// MongoDB Source → PostgreSQL Sink Tests +// ───────────────────────────────────────────────── + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_cdc_mode_mongodb_source_events() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + // MongoDB source has no real source DB for schema inference — we use + // SourceConnectionConfig::Mongodb which triggers canonical type mapping. + // But the PG sink in CDC mode needs a source_connection for auto-creating + // tables. Since we don't have a real MongoDB, pre-create the target table + // with the expected schema instead. + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"users\" ( + \"_cdc_op\" text, + \"_cdc_lsn\" bigint, + \"_cdc_timestamp_us\" bigint, + \"_cdc_snapshot\" boolean, + \"_cdc_schema\" text, + \"_cdc_table\" text, + \"_cdc_primary_key_columns\" text, + \"_id\" text, + \"name\" text, + \"age\" integer, + \"active\" boolean, + \"score\" double precision, + \"_old__id\" text, + \"_old_name\" text, + \"_old_age\" integer, + \"_old_active\" boolean, + \"_old_score\" double precision + )", + &[], + ) + .await + .unwrap(); + + let config = make_sink_config(&host, port); + let mut sink = PostgresSink::new(config, SinkMode::Cdc, None) + .await + .unwrap(); + + let events = vec![ + make_mongo_insert("demo", "users", "507f1f77bcf86cd799439011", "Alice"), + make_mongo_insert("demo", "users", "507f1f77bcf86cd799439012", "Bob"), + ]; + + sink.write_batch(&events).await.unwrap(); + sink.flush().await.unwrap(); + + // Verify rows landed with correct typed values + let rows = client + .query( + "SELECT \"_cdc_op\", \"_id\", \"name\", \"age\", \"active\", \"score\" \ + FROM \"public\".\"users\" ORDER BY \"_id\"", + &[], + ) + .await + .unwrap(); + + assert_eq!(rows.len(), 2); + + let op: &str = rows[0].get("_cdc_op"); + assert_eq!(op, "I"); + let id: Option<&str> = rows[0].get("_id"); + assert_eq!(id, Some("507f1f77bcf86cd799439011")); + let name: Option<&str> = rows[0].get("name"); + assert_eq!(name, Some("Alice")); + let age: Option = rows[0].get("age"); + assert_eq!(age, Some(30)); + let active: Option = rows[0].get("active"); + assert_eq!(active, Some(true)); + let score: Option = rows[0].get("score"); + assert_eq!(score, Some(95.5)); +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_replication_mode_mongodb_source_upsert_delete() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + // Pre-create table with MongoDB-style schema (_id as PK) + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"items\" ( + \"_id\" TEXT NOT NULL, + \"name\" TEXT, + \"price\" DOUBLE PRECISION, + PRIMARY KEY (\"_id\") + )", + &[], + ) + .await + .unwrap(); + + let config = make_sink_config(&host, port); + let mut sink = PostgresSink::new(config, SinkMode::Replication, None) + .await + .unwrap(); + + // Insert + let insert = CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text("item1".into())), + ("name".to_string(), ColumnValue::Text("Widget".into())), + ("price".to_string(), ColumnValue::float(19.99)), + ] + .into_iter() + .collect(), + ), + old: None, + primary_key_columns: vec!["_id".into()], + }; + sink.write_batch(&[insert]).await.unwrap(); + sink.flush().await.unwrap(); + + let count: i64 = client + .query_one("SELECT COUNT(*) FROM \"public\".\"items\"", &[]) + .await + .unwrap() + .get(0); + assert_eq!(count, 1); + + // Update (upsert) + let update = CdcEvent { + lsn: Lsn(200), + timestamp_us: 1_700_000_000_000_001, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Update, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text("item1".into())), + ("name".to_string(), ColumnValue::Text("Widget Pro".into())), + ("price".to_string(), ColumnValue::float(29.99)), + ] + .into_iter() + .collect(), + ), + old: Some( + vec![("_id".to_string(), ColumnValue::Text("item1".into()))] + .into_iter() + .collect(), + ), + primary_key_columns: vec!["_id".into()], + }; + sink.write_batch(&[update]).await.unwrap(); + sink.flush().await.unwrap(); + + // Still 1 row, but updated + let row = client + .query_one("SELECT \"name\", \"price\" FROM \"public\".\"items\"", &[]) + .await + .unwrap(); + let name: Option<&str> = row.get("name"); + assert_eq!(name, Some("Widget Pro")); + let price: Option = row.get("price"); + assert_eq!(price, Some(29.99)); + + // Delete + let delete = CdcEvent { + lsn: Lsn(300), + timestamp_us: 1_700_000_000_000_002, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Delete, + new: None, + old: Some( + vec![("_id".to_string(), ColumnValue::Text("item1".into()))] + .into_iter() + .collect(), + ), + primary_key_columns: vec!["_id".into()], + }; + sink.write_batch(&[delete]).await.unwrap(); + sink.flush().await.unwrap(); + + let count: i64 = client + .query_one("SELECT COUNT(*) FROM \"public\".\"items\"", &[]) + .await + .unwrap() + .get(0); + assert_eq!(count, 0); +} From d0ec4e775c741e702e9587fabc31b88915a4605a Mon Sep 17 00:00:00 2001 From: Manfred Lee Date: Thu, 26 Mar 2026 00:27:51 -0700 Subject: [PATCH 3/6] Upgrade MongoDB version from 7 to 8 in docker-compose and integration tests --- example/docker-compose.yml | 4 ++-- tests/mongodb_integration.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example/docker-compose.yml b/example/docker-compose.yml index a6261ab..a0e147e 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -51,7 +51,7 @@ services: retries: 10 mongodb: - image: mongo:7 + image: mongo:8 container_name: cdc-mongodb ports: - "27017:27017" @@ -63,7 +63,7 @@ services: retries: 10 mongodb-init: - image: mongo:7 + image: mongo:8 container_name: cdc-mongodb-init depends_on: mongodb: diff --git a/tests/mongodb_integration.rs b/tests/mongodb_integration.rs index e0b9ef7..297615b 100644 --- a/tests/mongodb_integration.rs +++ b/tests/mongodb_integration.rs @@ -59,7 +59,7 @@ impl Sink for CollectorSink { /// Start a MongoDB container configured as a single-node replica set. async fn start_mongodb() -> (ContainerAsync, String, u16) { let container = Mongo::default() - .with_tag("7") + .with_tag("8") .with_cmd(vec![ "mongod".to_string(), "--replSet".to_string(), From ae7d07c0b942a05fd149314d58c68b041e9555d9 Mon Sep 17 00:00:00 2001 From: Manfred Lee Date: Mon, 27 Jul 2026 01:20:24 -0700 Subject: [PATCH 4/6] Fix MongoDB source correctness bugs and add regression coverage Snapshot checkpoints were persisted as untagged JSON, and ResumeToken deserializes from any JSON object, so a restart after a completed snapshot fed the server a bogus resume token and never re-snapshotted. Offsets are now an externally tagged MongoOffset enum (snapshot_in_progress / snapshot_complete / token); a completed snapshot records its cluster time so a restart starts the change stream there, an interrupted one re-runs, and unreadable offsets fail loudly. Update and delete events dropped the primary key when a collection had no pre-images enabled (the MongoDB default): the Postgres sink errored with "no old row" and Iceberg equality deletes got a null _id. `old` now falls back to the change event's documentKey. CRUD events whose document could not be looked up are skipped instead of emitting a row-less change. Also: - pass the shutdown token into the snapshot so a long scan is cancellable, and treat a channel closed during shutdown as a clean stop - fail instead of exiting successfully when the server closes the change stream - commit one resume token per drained server batch (or 1000 events) rather than one offset write and sink flush per document - merge conflicting BSON types during schema inference (nulls ignored, numerics widened, otherwise text) in both the source and schema discovery - fix pre-1970 timestamps, prefer wallTime over clusterTime, drop a dead enum variant, and sort the discovery sample for stable schemas Examples and docs: fix a self-referencing `const db` that made the MongoDB init script throw, correct the namespace type in mongo-to-iceberg.json, align mongo-to-pg.json credentials, and document source requirements and pre-images. Tests: unit coverage for change event conversion, offset round-trips and BSON type merging, a guard that every example config deserializes, and integration tests for deletes without pre-images, snapshot-phase offset resume, collection filtering, shutdown mid-snapshot, and Mongo to Postgres replication. --- README.md | 20 + .../configs/mongo-to-iceberg-replication.json | 31 + example/configs/mongo-to-iceberg.json | 2 +- example/configs/mongo-to-pg-replication.json | 23 + example/configs/mongo-to-pg.json | 9 +- example/init/mongodb/init.js | 39 +- src/config/mod.rs | 22 + src/schema/discovery.rs | 21 +- src/schema/type_mapping.rs | 2 + src/source/mongodb/bson_mapping.rs | 164 ++++- src/source/mongodb/change_stream.rs | 112 ++- src/source/mongodb/event_converter.rs | 284 +++++++- src/source/mongodb/mod.rs | 71 +- src/source/mongodb/offset.rs | 160 +++++ src/source/mongodb/snapshot.rs | 112 ++- tests/mongodb_integration.rs | 646 +++++++++++++++++- 16 files changed, 1589 insertions(+), 129 deletions(-) create mode 100644 example/configs/mongo-to-iceberg-replication.json create mode 100644 example/configs/mongo-to-pg-replication.json create mode 100644 src/source/mongodb/offset.rs diff --git a/README.md b/README.md index 2100ddc..8e5b309 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,26 @@ checkpoints progress in the offset store. On restart, the pipeline resumes from - **Platform**: Linux and macOS only - **CMake**: Required for building the bundled librdkafka (Kafka dependency) +### Source requirements + +| Source | Requirements | +| --- | --- | +| PostgreSQL | Logical replication (`wal_level=logical`), a replication slot and publication | +| MySQL | Row-based binlog (`binlog_format=ROW`, `binlog_row_image=FULL`) | +| MongoDB | Replica set (or sharded cluster) — change streams do not work on standalone `mongod` | + +**MongoDB pre-images (optional).** Enabling `changeStreamPreAndPostImages` on a +collection makes the full "before" row available on updates and deletes: + +```js +db.runCommand({ collMod: "users", changeStreamPreAndPostImages: { enabled: true } }); +``` + +Without it, update and delete events fall back to the change event's +`documentKey`, so the `_id` primary key is always captured — replication mode +still applies updates and deletes correctly, but CDC-mode `old` values contain +only `_id`. + ## Quick Start - Standalone Mode The standalone mode runs a single pipeline without the admin server. This is ideal for local testing and development. diff --git a/example/configs/mongo-to-iceberg-replication.json b/example/configs/mongo-to-iceberg-replication.json new file mode 100644 index 0000000..e67ccb5 --- /dev/null +++ b/example/configs/mongo-to-iceberg-replication.json @@ -0,0 +1,31 @@ +{ + "mode": "replication", + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "iceberg", + "catalog": { + "type": "rest", + "uri": "http://localhost:8181", + "warehouse": "s3://warehouse/iceberg", + "properties": { + "s3.endpoint": "http://localhost:9000", + "s3.access-key-id": "minioadmin", + "s3.secret-access-key": "minioadmin", + "s3.path-style-access": "true", + "s3.region": "us-east-1" + } + }, + "namespace": ["demo"], + "table_prefix": "" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-iceberg-replication" + } +} diff --git a/example/configs/mongo-to-iceberg.json b/example/configs/mongo-to-iceberg.json index d4a41ad..29c8d63 100644 --- a/example/configs/mongo-to-iceberg.json +++ b/example/configs/mongo-to-iceberg.json @@ -19,7 +19,7 @@ "s3.region": "us-east-1" } }, - "namespace": "demo", + "namespace": ["demo"], "table_prefix": "" }, "offset": { diff --git a/example/configs/mongo-to-pg-replication.json b/example/configs/mongo-to-pg-replication.json new file mode 100644 index 0000000..08c15c3 --- /dev/null +++ b/example/configs/mongo-to-pg-replication.json @@ -0,0 +1,23 @@ +{ + "mode": "replication", + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "postgres", + "host": "localhost", + "port": 5433, + "user": "cdc_user", + "password": "cdc_password", + "database": "demo_dest", + "schema": "public" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-pg-replication" + } +} diff --git a/example/configs/mongo-to-pg.json b/example/configs/mongo-to-pg.json index 72cbf46..f19c5cc 100644 --- a/example/configs/mongo-to-pg.json +++ b/example/configs/mongo-to-pg.json @@ -9,11 +9,10 @@ "type": "postgres", "host": "localhost", "port": 5433, - "user": "postgres", - "password": "postgres", - "database": "cdc_target", - "schema": "public", - "table_prefix": "" + "user": "cdc_user", + "password": "cdc_password", + "database": "demo_dest", + "schema": "public" }, "offset": { "type": "sqlite", diff --git a/example/init/mongodb/init.js b/example/init/mongodb/init.js index dc67a4e..f00314a 100644 --- a/example/init/mongodb/init.js +++ b/example/init/mongodb/init.js @@ -1,41 +1,56 @@ -// Initialize replica set (required for change streams) -rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost:27017" }] }); +// Initialize replica set (required for change streams). +// Re-running the script against an already-initialized set is not an error. +try { + rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost:27017" }] }); +} catch (e) { + print("replSetInitiate: " + e); +} // Wait for primary election let attempts = 0; -while (!rs.isMaster().ismaster && attempts < 30) { +while (!db.hello().isWritablePrimary && attempts < 30) { sleep(1000); attempts++; } -if (!rs.isMaster().ismaster) { +if (!db.hello().isWritablePrimary) { print("ERROR: Failed to elect primary after 30 seconds"); quit(1); } print("Replica set initialized, primary elected"); -// Create demo database and collections -const db = db.getSiblingDB("demo"); +// Create demo database and collections. +// NOTE: `const db = db.getSiblingDB(...)` is a self-referencing declaration and +// throws a ReferenceError — the handle must use a different name. +const demoDb = db.getSiblingDB("demo"); -// Enable pre/post images for change streams (MongoDB 6.0+) -db.createCollection("users", { +// Enable pre/post images for change streams (MongoDB 6.0+). +// Optional: without them, update/delete events fall back to `documentKey`, so +// the `_id` primary key is still captured but the full "before" row is not. +demoDb.createCollection("users", { changeStreamPreAndPostImages: { enabled: true }, }); -db.createCollection("orders", { +demoDb.createCollection("orders", { changeStreamPreAndPostImages: { enabled: true }, }); // Insert sample data -db.users.insertMany([ +demoDb.users.insertMany([ { _id: ObjectId(), name: "Alice", email: "alice@example.com", age: 30, active: true }, { _id: ObjectId(), name: "Bob", email: "bob@example.com", age: 25, active: true }, { _id: ObjectId(), name: "Charlie", email: "charlie@example.com", age: 35, active: false }, ]); -db.orders.insertMany([ +demoDb.orders.insertMany([ { _id: ObjectId(), user: "Alice", product: "Widget", quantity: 2, price: 19.99 }, { _id: ObjectId(), user: "Bob", product: "Gadget", quantity: 1, price: 49.99 }, ]); -print("Demo data inserted: " + db.users.countDocuments() + " users, " + db.orders.countDocuments() + " orders"); +print( + "Demo data inserted: " + + demoDb.users.countDocuments() + + " users, " + + demoDb.orders.countDocuments() + + " orders", +); diff --git a/src/config/mod.rs b/src/config/mod.rs index 0dfb246..08a9c13 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -57,6 +57,28 @@ pub struct Config { mod tests { use super::*; + /// Every shipped example config must deserialize, so a config that drifts + /// from the schema (wrong field type, renamed key) fails the build rather + /// than only failing when a user runs it. + #[test] + fn test_example_configs_deserialize() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("example/configs"); + let mut checked = 0; + + for entry in std::fs::read_dir(&dir).expect("example/configs must exist") { + let path = entry.expect("readable dir entry").path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + let contents = std::fs::read_to_string(&path).expect("readable config"); + serde_json::from_str::(&contents) + .unwrap_or_else(|e| panic!("{} failed to parse: {e}", path.display())); + checked += 1; + } + + assert!(checked > 0, "no example configs found in {}", dir.display()); + } + #[test] fn test_sink_mode_default_is_cdc() { let json = r#"{ diff --git a/src/schema/discovery.rs b/src/schema/discovery.rs index 3ec8c3e..90754fd 100644 --- a/src/schema/discovery.rs +++ b/src/schema/discovery.rs @@ -346,11 +346,15 @@ async fn fetch_column_subset_mysql( Ok(result) } +/// Number of documents sampled to infer a MongoDB collection's schema. +const SCHEMA_SAMPLE_SIZE: i64 = 100; + /// Fetch columns by sampling documents from a MongoDB collection. /// -/// Since MongoDB is schemaless, we sample up to 100 documents and build a -/// union of all top-level field names, inferring BSON type from the first -/// non-null value seen per field. +/// Since MongoDB is schemaless, we sample up to `SCHEMA_SAMPLE_SIZE` documents +/// and build a union of all top-level field names. Types observed for the same +/// field across documents are merged (nulls ignored, numerics widened, +/// otherwise falling back to text). async fn fetch_columns_mongodb( url: &str, database: &str, @@ -365,9 +369,12 @@ async fn fetch_columns_mongodb( let db = client.database(database); let coll = db.collection::(collection); + // Sort by `_id` so repeated runs sample the same documents and infer a + // stable schema instead of whatever the server happens to return first. let mut cursor = coll .find(doc! {}) - .limit(100) + .sort(doc! { "_id": 1 }) + .limit(SCHEMA_SAMPLE_SIZE) .await .map_err(|e| CdcError::Schema(format!("mongodb find: {e}")))?; @@ -377,11 +384,7 @@ async fn fetch_columns_mongodb( use futures_util::StreamExt; while let Some(result) = cursor.next().await { let doc = result.map_err(|e| CdcError::Schema(format!("mongodb cursor: {e}")))?; - for (key, value) in &doc { - field_types.entry(key.clone()).or_insert_with(|| { - crate::source::mongodb::bson_mapping::bson_type_string(value).to_string() - }); - } + crate::source::mongodb::bson_mapping::merge_document_field_types(&mut field_types, &doc); } let dialect = SourceDialect::Mongodb; diff --git a/src/schema/type_mapping.rs b/src/schema/type_mapping.rs index b88a6ca..da6b815 100644 --- a/src/schema/type_mapping.rs +++ b/src/schema/type_mapping.rs @@ -155,6 +155,8 @@ fn parse_mongodb_type(dt: &str) -> CanonicalType { "bindata" => CanonicalType::Binary, "object" | "array" => CanonicalType::Json, "decimal" => CanonicalType::Text, // Decimal128 has 34 digits precision, store as text + // A field that was null in every sampled document carries no type info. + "null" => CanonicalType::Text, _ => CanonicalType::Text, } } diff --git a/src/source/mongodb/bson_mapping.rs b/src/source/mongodb/bson_mapping.rs index 2780294..2b8c88a 100644 --- a/src/source/mongodb/bson_mapping.rs +++ b/src/source/mongodb/bson_mapping.rs @@ -18,11 +18,9 @@ pub fn bson_to_column_value(bson: &Bson) -> ColumnValue { Bson::String(s) => ColumnValue::Text(s.clone()), Bson::ObjectId(oid) => ColumnValue::Text(oid.to_hex()), Bson::DateTime(dt) => { - // Convert millis to ISO 8601 via chrono - let millis = dt.timestamp_millis(); - let secs = millis / 1000; - let nsecs = ((millis % 1000) * 1_000_000) as u32; - let ts = chrono::DateTime::from_timestamp(secs, nsecs) + // Convert millis to ISO 8601 via chrono. `from_timestamp_millis` handles + // negative (pre-1970) values correctly — manual secs/nanos splitting does not. + let ts = chrono::DateTime::from_timestamp_millis(dt.timestamp_millis()) .unwrap_or_default() .to_rfc3339(); ColumnValue::Timestamp(ts) @@ -78,16 +76,54 @@ pub fn bson_type_string(bson: &Bson) -> &'static str { } } +/// Merge two observed BSON type names for the same field into a single type +/// that can hold both. +/// +/// MongoDB is schemaless, so the same field can hold different types across +/// documents. Rules: +/// - `null` carries no type information — the other type wins. +/// - Identical types stay as-is. +/// - Integer widening: `int` + `long` → `long`. +/// - Numeric widening: any int type + `double` → `double`. +/// - Anything else conflicting falls back to `string`, which maps to a text +/// column and can hold any stringified value. +pub fn merge_bson_types(existing: &str, incoming: &str) -> String { + if existing == incoming { + return existing.to_string(); + } + match (existing, incoming) { + ("null", other) | (other, "null") => other.to_string(), + ("int", "long") | ("long", "int") => "long".to_string(), + ("int", "double") | ("double", "int") => "double".to_string(), + ("long", "double") | ("double", "long") => "double".to_string(), + ("object", "array") | ("array", "object") => "object".to_string(), + _ => "string".to_string(), + } +} + +/// Fold a document's top-level fields into an accumulating field → type map, +/// merging types when a field has already been seen with a different type. +pub fn merge_document_field_types(field_types: &mut BTreeMap, doc: &Document) { + for (key, value) in doc { + let incoming = bson_type_string(value); + match field_types.get_mut(key) { + Some(existing) => { + *existing = merge_bson_types(existing, incoming); + } + None => { + field_types.insert(key.clone(), incoming.to_string()); + } + } + } +} + /// Infer schema from a set of documents by building a union of all top-level -/// fields. The type for each field is taken from the first non-null value seen. +/// fields. Types observed across documents are merged (see [`merge_bson_types`]), +/// so a field that is null in one document and an int in another is typed `int`. pub fn infer_schema_from_documents(docs: &[Document]) -> Vec<(String, String)> { let mut field_types: BTreeMap = BTreeMap::new(); for doc in docs { - for (key, value) in doc { - field_types - .entry(key.clone()) - .or_insert_with(|| bson_type_string(value).to_string()); - } + merge_document_field_types(&mut field_types, doc); } field_types.into_iter().collect() } @@ -268,4 +304,110 @@ mod tests { let schema = infer_schema_from_documents(&[]); assert!(schema.is_empty()); } + + #[test] + fn test_bson_to_column_value_timestamp() { + let ts = mongodb::bson::Timestamp { + time: 1700000000, + increment: 5, + }; + let result = bson_to_column_value(&Bson::Timestamp(ts)); + assert_eq!(result, ColumnValue::Text("1700000000:5".into())); + } + + #[test] + fn test_bson_to_column_value_datetime_before_epoch() { + // 1969-12-31T23:59:59.500Z — negative millis with a fractional part. + let dt = mongodb::bson::DateTime::from_millis(-500); + let result = bson_to_column_value(&Bson::DateTime(dt)); + match result { + ColumnValue::Timestamp(s) => { + assert!( + s.starts_with("1969-12-31T23:59:59.5"), + "expected a pre-epoch timestamp, got {s}" + ); + } + other => panic!("expected Timestamp, got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_min_max_key_fallback() { + for bson in [Bson::MinKey, Bson::MaxKey, Bson::Undefined] { + assert!( + matches!(bson_to_column_value(&bson), ColumnValue::Text(_)), + "expected Text fallback for {bson:?}" + ); + } + } + + #[test] + fn test_bson_to_column_value_empty_binary() { + let bin = Binary { + subtype: mongodb::bson::spec::BinarySubtype::Generic, + bytes: vec![], + }; + assert_eq!( + bson_to_column_value(&Bson::Binary(bin)), + ColumnValue::Bytes(vec![]) + ); + } + + #[test] + fn test_merge_bson_types_null_is_ignored() { + assert_eq!(merge_bson_types("null", "int"), "int"); + assert_eq!(merge_bson_types("int", "null"), "int"); + assert_eq!(merge_bson_types("null", "null"), "null"); + } + + #[test] + fn test_merge_bson_types_numeric_widening() { + let cases = vec![ + (("int", "long"), "long"), + (("long", "int"), "long"), + (("int", "double"), "double"), + (("double", "long"), "double"), + ]; + for ((a, b), expected) in cases { + assert_eq!(merge_bson_types(a, b), expected, "for ({a}, {b})"); + } + } + + #[test] + fn test_merge_bson_types_conflict_falls_back_to_string() { + assert_eq!(merge_bson_types("int", "bool"), "string"); + assert_eq!(merge_bson_types("date", "objectid"), "string"); + assert_eq!(merge_bson_types("object", "array"), "object"); + } + + #[test] + fn test_merge_bson_types_identical() { + assert_eq!(merge_bson_types("string", "string"), "string"); + } + + #[test] + fn test_infer_schema_ignores_null_when_typing_field() { + let docs = vec![doc! { "score": Bson::Null }, doc! { "score": 42i32 }]; + let schema = infer_schema_from_documents(&docs); + assert_eq!(schema, vec![("score".to_string(), "int".to_string())]); + } + + #[test] + fn test_infer_schema_widens_conflicting_types() { + let docs = vec![ + doc! { "id": 1i32, "v": 1i32 }, + doc! { "id": "a", "v": 2i64 }, + ]; + let type_map: BTreeMap = + infer_schema_from_documents(&docs).into_iter().collect(); + assert_eq!(type_map["id"], "string"); // int + string → text column + assert_eq!(type_map["v"], "long"); // int + long → long + } + + #[test] + fn test_infer_schema_all_null_field_stays_null() { + let docs = vec![doc! { "x": Bson::Null }, doc! { "x": Bson::Null }]; + let schema = infer_schema_from_documents(&docs); + assert_eq!(schema, vec![("x".to_string(), "null".to_string())]); + } } diff --git a/src/source/mongodb/change_stream.rs b/src/source/mongodb/change_stream.rs index 835d453..9ec2d4c 100644 --- a/src/source/mongodb/change_stream.rs +++ b/src/source/mongodb/change_stream.rs @@ -1,4 +1,5 @@ -use futures_util::StreamExt; +use std::time::Duration; + use mongodb::bson::{doc, Document, Timestamp}; use mongodb::change_stream::event::ResumeToken; use mongodb::options::FullDocumentBeforeChangeType; @@ -10,7 +11,16 @@ use tokio_util::sync::CancellationToken; use crate::error::{CdcError, Result}; use crate::source::SourceEvent; -use super::event_converter::{change_event_to_cdc_event, serialize_resume_token, ConvertResult}; +use super::event_converter::{change_event_to_cdc_event, ConvertResult}; +use super::offset::MongoOffset; +use super::{send_event, SendOutcome}; + +/// Max change events buffered before forcing an offset commit. +const MAX_EVENTS_PER_COMMIT: usize = 1000; + +/// How long a `getMore` waits server-side for new events before returning empty. +/// Bounds the latency of the commit that flushes a partially filled batch. +const MAX_AWAIT_TIME: Duration = Duration::from_millis(500); /// Determines where to start the change stream from. pub enum ResumePoint { @@ -25,6 +35,10 @@ pub enum ResumePoint { /// Supports filtering to specific collections via a `$match` pipeline stage. /// Uses `fullDocument: "updateLookup"` to get the full document on updates, /// and `fullDocumentBeforeChange: "whenAvailable"` to capture pre-images. +/// +/// Events are drained batch-by-batch: a `Commit` carrying the latest resume +/// token is emitted once a server batch is exhausted (or after +/// `MAX_EVENTS_PER_COMMIT` events), rather than once per document. pub async fn run_change_stream( client: &Client, database: &str, @@ -50,7 +64,8 @@ pub async fn run_change_stream( .watch() .pipeline(pipeline) .full_document(FullDocumentType::UpdateLookup) - .full_document_before_change(FullDocumentBeforeChangeType::WhenAvailable); + .full_document_before_change(FullDocumentBeforeChangeType::WhenAvailable) + .max_await_time(MAX_AWAIT_TIME); match &resume_point { ResumePoint::AfterSnapshot { snapshot_time } => { @@ -67,44 +82,79 @@ pub async fn run_change_stream( tracing::info!(database, "change stream started"); + // Latest resume token not yet committed, and how many events it covers. + let mut pending_offset: Option = None; + let mut pending_events: usize = 0; + loop { - tokio::select! { + let next = tokio::select! { _ = shutdown.cancelled() => { tracing::info!("shutdown received, stopping change stream"); + commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown).await?; return Ok(()); } - next = stream.next() => { - match next { - Some(Ok(event)) => { - match change_event_to_cdc_event(&event, database)? { - ConvertResult::Event(cdc_event) => { - sender - .send(SourceEvent::Change(cdc_event)) - .await - .map_err(|e| CdcError::Mongodb(format!("failed to send change event: {e}")))?; - - // Send commit with resume token as offset - let offset = serialize_resume_token(&event.id)?; - sender - .send(SourceEvent::Commit { offset }) - .await - .map_err(|e| CdcError::Mongodb(format!("failed to send commit: {e}")))?; - } - ConvertResult::Skip => { - tracing::debug!(op_type = ?event.operation_type, "skipping unsupported change event"); - } - ConvertResult::Commit { .. } => {} + // `next_if_any` performs at most one `getMore`, so an empty result + // means the current batch is drained — the point at which the driver + // documentation recommends persisting the resume token. + next = stream.next_if_any() => next, + }; + + match next { + Ok(Some(event)) => { + // Track the token even for skipped events so the stream position + // still advances past operations we do not forward. + let offset = MongoOffset::Token(event.id.clone()).to_json()?; + + match change_event_to_cdc_event(&event, database)? { + ConvertResult::Event(cdc_event) => { + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Change(cdc_event), shutdown).await? + { + return Ok(()); } } - Some(Err(e)) => { - return Err(CdcError::Mongodb(format!("change stream error: {e}"))); - } - None => { - tracing::warn!("change stream ended unexpectedly"); - return Ok(()); + ConvertResult::Skip => { + tracing::debug!(op_type = ?event.operation_type, "skipping unsupported change event"); } } + + pending_offset = Some(offset); + pending_events += 1; + + if pending_events >= MAX_EVENTS_PER_COMMIT { + commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown) + .await?; + } + } + Ok(None) => { + // Batch drained: commit what we have, then check the stream is alive. + commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown).await?; + + if !stream.is_alive() { + return Err(CdcError::Mongodb( + "change stream closed by the server (the database or a watched collection \ + was likely dropped or renamed)" + .into(), + )); + } + } + Err(e) => { + return Err(CdcError::Mongodb(format!("change stream error: {e}"))); } } } } + +/// Emit a `Commit` for the pending resume token, if any. +async fn commit_pending( + sender: &mpsc::Sender, + pending_offset: &mut Option, + pending_events: &mut usize, + shutdown: &CancellationToken, +) -> Result<()> { + if let Some(offset) = pending_offset.take() { + send_event(sender, SourceEvent::Commit { offset }, shutdown).await?; + } + *pending_events = 0; + Ok(()) +} diff --git a/src/source/mongodb/event_converter.rs b/src/source/mongodb/event_converter.rs index 6840b15..1045321 100644 --- a/src/source/mongodb/event_converter.rs +++ b/src/source/mongodb/event_converter.rs @@ -1,7 +1,7 @@ use mongodb::bson::Document; -use mongodb::change_stream::event::{ChangeStreamEvent, ResumeToken}; +use mongodb::change_stream::event::ChangeStreamEvent; -use crate::error::{CdcError, Result}; +use crate::error::Result; use crate::event::{CdcEvent, ChangeOp, Lsn, TableId}; use super::bson_mapping::document_to_row; @@ -10,9 +10,8 @@ use super::bson_mapping::document_to_row; pub enum ConvertResult { /// A CDC event to send downstream. Event(CdcEvent), - /// A commit with serialized resume token as offset. - Commit { offset: String }, - /// Skip this event (unsupported operation type). + /// Skip this event (unsupported operation type, or a CRUD event whose + /// document could no longer be looked up). Skip, } @@ -20,6 +19,11 @@ pub enum ConvertResult { /// /// Returns `ConvertResult::Event` for insert/update/replace/delete, /// `ConvertResult::Skip` for unsupported operations (drop, rename, etc.). +/// +/// `old` is populated from the pre-image when the collection has +/// `changeStreamPreAndPostImages` enabled, and otherwise falls back to +/// `documentKey` so that update/delete events always carry the `_id` primary +/// key that sinks need to target the right row. pub fn change_event_to_cdc_event( event: &ChangeStreamEvent, database: &str, @@ -38,17 +42,46 @@ pub fn change_event_to_cdc_event( .and_then(|ns| ns.coll.as_deref()) .unwrap_or("unknown"); + // Prefer wall time (millisecond resolution) over cluster time (second resolution). let timestamp_us = event - .cluster_time - .map(|ts| ts.time as i64 * 1_000_000) - .or_else(|| event.wall_time.map(|wt| wt.timestamp_millis() * 1000)) + .wall_time + .map(|wt| wt.timestamp_millis() * 1000) + .or_else(|| event.cluster_time.map(|ts| ts.time as i64 * 1_000_000)) .unwrap_or(0); + let document_key_row = event.document_key.as_ref().map(document_to_row); + let new = event.full_document.as_ref().map(document_to_row); - let old = event - .full_document_before_change - .as_ref() - .map(document_to_row); + let old = match op { + ChangeOp::Update | ChangeOp::Delete => event + .full_document_before_change + .as_ref() + .map(document_to_row) + .or_else(|| document_key_row.clone()), + _ => None, + }; + + // An insert/update/replace with no full document means the document was + // removed between the change and the `updateLookup`. A later delete event + // covers it, so skip rather than emitting a row-less change that sinks + // cannot apply. + if matches!(op, ChangeOp::Insert | ChangeOp::Update) && new.is_none() { + tracing::warn!( + collection, + op_type = ?event.operation_type, + "change event has no full document (document removed before lookup), skipping" + ); + return Ok(ConvertResult::Skip); + } + + // A delete with neither pre-image nor documentKey has no way to identify the row. + if op == ChangeOp::Delete && old.is_none() { + tracing::warn!( + collection, + "delete event has neither pre-image nor documentKey, skipping" + ); + return Ok(ConvertResult::Skip); + } let cdc_event = CdcEvent { lsn: Lsn(0), @@ -87,22 +120,223 @@ pub fn snapshot_doc_to_cdc_event(doc: &Document, database: &str, collection: &st } } -/// Serialize a resume token to a JSON string for offset storage. -pub fn serialize_resume_token(token: &ResumeToken) -> Result { - serde_json::to_string(token) - .map_err(|e| CdcError::Mongodb(format!("failed to serialize resume token: {e}"))) -} - -/// Deserialize a resume token from a JSON string. -pub fn deserialize_resume_token(s: &str) -> Result { - serde_json::from_str(s) - .map_err(|e| CdcError::Mongodb(format!("failed to deserialize resume token: {e}"))) -} - #[cfg(test)] mod tests { use super::*; - use mongodb::bson::doc; + use crate::event::ColumnValue; + use mongodb::bson::{doc, Timestamp}; + + /// Build a `ChangeStreamEvent` from a raw change event document, the same + /// way the driver deserializes events off the wire. + fn event_from_doc(doc: Document) -> ChangeStreamEvent { + mongodb::bson::from_document(doc).expect("failed to deserialize change event") + } + + /// A minimal change event document with the given operation type. + fn base_event(op_type: &str, coll: &str) -> Document { + doc! { + "_id": { "_data": "8265F1A2B3000000012B" }, + "operationType": op_type, + "ns": { "db": "mydb", "coll": coll }, + "clusterTime": Timestamp { time: 1_700_000_000, increment: 1 }, + } + } + + fn expect_event(result: ConvertResult) -> CdcEvent { + match result { + ConvertResult::Event(e) => e, + ConvertResult::Skip => panic!("expected Event, got Skip"), + } + } + + #[test] + fn test_insert_event_maps_to_insert() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Alice" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Insert); + assert_eq!(event.table.schema, "mydb"); + assert_eq!(event.table.name, "users"); + assert_eq!( + event.new.as_ref().and_then(|r| r.get("name")), + Some(&ColumnValue::Text("Alice".into())) + ); + assert!(event.old.is_none(), "insert must not carry an old row"); + } + + #[test] + fn test_update_event_maps_to_update_with_full_document() { + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Alice2" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Update); + assert_eq!( + event.new.as_ref().and_then(|r| r.get("name")), + Some(&ColumnValue::Text("Alice2".into())) + ); + } + + #[test] + fn test_replace_event_maps_to_update() { + let mut raw = base_event("replace", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Replaced" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Update); + } + + #[test] + fn test_update_without_pre_image_uses_document_key_as_old() { + // Pre-images are disabled by default in MongoDB; sinks still need the + // primary key to target the row being replaced. + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Alice2" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + let old = event + .old + .expect("update should carry an old row with the pk"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("u1".into()))); + } + + #[test] + fn test_update_prefers_pre_image_over_document_key() { + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "new" }); + raw.insert( + "fullDocumentBeforeChange", + doc! { "_id": "u1", "name": "old" }, + ); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + let old = event.old.expect("update should carry a pre-image"); + assert_eq!(old.get("name"), Some(&ColumnValue::Text("old".into()))); + } + + #[test] + fn test_delete_without_pre_image_uses_document_key_as_old() { + let mut raw = base_event("delete", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Delete); + assert!(event.new.is_none()); + let old = event.old.expect("delete must carry an old row with the pk"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("u1".into()))); + } + + #[test] + fn test_delete_with_pre_image_carries_full_old_row() { + let mut raw = base_event("delete", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert( + "fullDocumentBeforeChange", + doc! { "_id": "u1", "name": "Alice" }, + ); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + let old = event.old.unwrap(); + assert_eq!(old.get("name"), Some(&ColumnValue::Text("Alice".into()))); + } + + #[test] + fn test_delete_without_document_key_is_skipped() { + let raw = base_event("delete", "users"); + + let result = change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap(); + + assert!( + matches!(result, ConvertResult::Skip), + "a delete with no way to identify the row must be skipped" + ); + } + + #[test] + fn test_update_without_full_document_is_skipped() { + // The document was deleted before `updateLookup` could resolve it. + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + + let result = change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap(); + + assert!(matches!(result, ConvertResult::Skip)); + } + + #[test] + fn test_unsupported_operation_types_are_skipped() { + for op_type in ["drop", "rename", "dropDatabase", "invalidate"] { + let raw = base_event(op_type, "users"); + let result = change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap(); + assert!( + matches!(result, ConvertResult::Skip), + "expected {op_type} to be skipped" + ); + } + } + + #[test] + fn test_missing_namespace_falls_back_to_unknown_collection() { + let mut raw = base_event("insert", "users"); + raw.remove("ns"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.table.name, "unknown"); + } + + #[test] + fn test_wall_time_is_preferred_over_cluster_time() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + // Wall time has millisecond resolution; cluster time only seconds. + raw.insert( + "wallTime", + mongodb::bson::DateTime::from_millis(1_700_000_000_123), + ); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.timestamp_us, 1_700_000_000_123 * 1000); + } + + #[test] + fn test_cluster_time_used_when_wall_time_missing() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.timestamp_us, 1_700_000_000 * 1_000_000); + } + + #[test] + fn test_primary_key_is_always_id() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.primary_key_columns, vec!["_id".to_string()]); + } #[test] fn test_snapshot_doc_to_cdc_event() { diff --git a/src/source/mongodb/mod.rs b/src/source/mongodb/mod.rs index 467e7e5..e2d8d11 100644 --- a/src/source/mongodb/mod.rs +++ b/src/source/mongodb/mod.rs @@ -1,6 +1,7 @@ pub mod bson_mapping; pub mod change_stream; pub mod event_converter; +pub mod offset; pub mod snapshot; use tokio::sync::mpsc; @@ -12,9 +13,39 @@ use crate::offset::OffsetStore; use crate::source::Source; use self::change_stream::{run_change_stream, ResumePoint}; -use self::event_converter::deserialize_resume_token; +use self::offset::MongoOffset; use self::snapshot::perform_snapshot; +/// Outcome of sending an event to the pipeline. +pub(crate) enum SendOutcome { + Sent, + /// The pipeline stopped receiving while a shutdown was in flight. + ShuttingDown, +} + +/// Send an event to the pipeline, treating a closed channel during shutdown as +/// a clean stop rather than an error. +/// +/// The pipeline's receive loop exits as soon as the shutdown token fires, which +/// can happen between a source's own cancellation checks and its next send. +pub(crate) async fn send_event( + sender: &mpsc::Sender, + event: crate::source::SourceEvent, + shutdown: &CancellationToken, +) -> Result { + match sender.send(event).await { + Ok(()) => Ok(SendOutcome::Sent), + Err(e) => { + if shutdown.is_cancelled() { + tracing::info!("pipeline stopped receiving during shutdown"); + Ok(SendOutcome::ShuttingDown) + } else { + Err(CdcError::Mongodb(format!("failed to send event: {e}"))) + } + } + } +} + /// MongoDB CDC source implementing the Source trait. /// /// Orchestrates: check offset → snapshot if needed → stream change events. @@ -46,23 +77,47 @@ impl Source for MongodbSource { // Phase 1: Determine start position (snapshot or resume) let saved_offset = self.offset_store.load().await?; - let resume_point = match saved_offset { - Some(offset_str) => { - tracing::info!("resuming from saved offset, skipping snapshot"); - let token = deserialize_resume_token(&offset_str)?; + let stored = match saved_offset { + Some(offset_str) => Some(MongoOffset::from_json(&offset_str)?), + None => None, + }; + + let resume_point = match stored { + Some(MongoOffset::Token(token)) => { + tracing::info!("resuming change stream from saved resume token"); ResumePoint::Token(token) } - None => { - tracing::info!("no saved offset, performing initial snapshot"); + Some(MongoOffset::SnapshotComplete { cluster_time }) => { + tracing::info!( + cluster_time = %format!("{}:{}", cluster_time.time, cluster_time.increment), + "snapshot already complete, starting change stream at recorded cluster time" + ); + ResumePoint::AfterSnapshot { + snapshot_time: cluster_time.into(), + } + } + other => { + match &other { + Some(MongoOffset::SnapshotInProgress { collection, count }) => { + tracing::warn!( + %collection, + count, + "previous snapshot was interrupted; snapshots are not resumable, re-running it" + ); + } + _ => tracing::info!("no saved offset, performing initial snapshot"), + } + let result = perform_snapshot( &client, &self.config.database, &self.config.collections, &sender, + &shutdown, ) .await?; - if shutdown.is_cancelled() { + if !result.completed { return Ok(()); } diff --git a/src/source/mongodb/offset.rs b/src/source/mongodb/offset.rs new file mode 100644 index 0000000..ebaaa95 --- /dev/null +++ b/src/source/mongodb/offset.rs @@ -0,0 +1,160 @@ +use mongodb::bson::Timestamp; +use mongodb::change_stream::event::ResumeToken; +use serde::{Deserialize, Serialize}; + +use crate::error::{CdcError, Result}; + +/// A cluster time (BSON timestamp) in a form that round-trips through plain JSON. +/// +/// `bson::Timestamp`'s own serde representation depends on the serializer, so the +/// two components are stored explicitly instead. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClusterTime { + pub time: u32, + pub increment: u32, +} + +impl From for ClusterTime { + fn from(ts: Timestamp) -> Self { + Self { + time: ts.time, + increment: ts.increment, + } + } +} + +impl From for Timestamp { + fn from(ct: ClusterTime) -> Self { + Timestamp { + time: ct.time, + increment: ct.increment, + } + } +} + +/// The persisted offset for a MongoDB source. +/// +/// Externally tagged so every stored offset carries an unambiguous kind. A bare +/// resume token must never be stored on its own: `ResumeToken` deserializes from +/// *any* JSON object, so an untagged offset from the snapshot phase would silently +/// be accepted as a (bogus) resume token and rejected by the server on resume. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MongoOffset { + /// Snapshot is partially done. Snapshots are not resumable mid-way, so a + /// restart from this offset re-runs the whole snapshot. + SnapshotInProgress { collection: String, count: u64 }, + /// Snapshot finished for all collections. The change stream starts at the + /// cluster time recorded before the snapshot began. + SnapshotComplete { cluster_time: ClusterTime }, + /// Streaming position: an opaque change stream resume token. + Token(ResumeToken), +} + +impl MongoOffset { + /// Serialize to the JSON string persisted by the offset store. + pub fn to_json(&self) -> Result { + serde_json::to_string(self) + .map_err(|e| CdcError::Mongodb(format!("failed to serialize offset: {e}"))) + } + + /// Parse an offset previously produced by [`MongoOffset::to_json`]. + pub fn from_json(s: &str) -> Result { + serde_json::from_str(s).map_err(|e| { + CdcError::Mongodb(format!( + "failed to parse stored offset ({e}); offset value: {s}" + )) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_token() -> ResumeToken { + // Resume tokens are opaque documents of the form {"_data": ""}. + serde_json::from_str(r#"{"_data":"8265F1A2B3000000012B"}"#).unwrap() + } + + #[test] + fn test_token_offset_round_trip() { + let offset = MongoOffset::Token(sample_token()); + let json = offset.to_json().unwrap(); + + match MongoOffset::from_json(&json).unwrap() { + MongoOffset::Token(token) => { + assert_eq!( + serde_json::to_string(&token).unwrap(), + serde_json::to_string(&sample_token()).unwrap() + ); + } + other => panic!("expected Token, got {other:?}"), + } + } + + #[test] + fn test_snapshot_complete_round_trip() { + let offset = MongoOffset::SnapshotComplete { + cluster_time: ClusterTime { + time: 1700000000, + increment: 7, + }, + }; + let json = offset.to_json().unwrap(); + + match MongoOffset::from_json(&json).unwrap() { + MongoOffset::SnapshotComplete { cluster_time } => { + assert_eq!(cluster_time.time, 1700000000); + assert_eq!(cluster_time.increment, 7); + } + other => panic!("expected SnapshotComplete, got {other:?}"), + } + } + + #[test] + fn test_snapshot_in_progress_round_trip() { + let offset = MongoOffset::SnapshotInProgress { + collection: "users".into(), + count: 1000, + }; + let json = offset.to_json().unwrap(); + + match MongoOffset::from_json(&json).unwrap() { + MongoOffset::SnapshotInProgress { collection, count } => { + assert_eq!(collection, "users"); + assert_eq!(count, 1000); + } + other => panic!("expected SnapshotInProgress, got {other:?}"), + } + } + + #[test] + fn test_snapshot_offset_is_not_parsed_as_resume_token() { + // The old (buggy) snapshot offset format must be rejected rather than + // silently accepted as a resume token. + let legacy = r#"{"snapshot_complete":"users","count":42}"#; + assert!(MongoOffset::from_json(legacy).is_err()); + } + + #[test] + fn test_arbitrary_json_is_rejected() { + for input in ["", "null", "\"token\"", "{}", r#"{"unknown":{}}"#] { + assert!( + MongoOffset::from_json(input).is_err(), + "expected rejection for {input}" + ); + } + } + + #[test] + fn test_cluster_time_timestamp_conversion() { + let ts = Timestamp { + time: 42, + increment: 3, + }; + let round_tripped: Timestamp = ClusterTime::from(ts).into(); + assert_eq!(round_tripped.time, 42); + assert_eq!(round_tripped.increment, 3); + } +} diff --git a/src/source/mongodb/snapshot.rs b/src/source/mongodb/snapshot.rs index 0079e22..122a9b9 100644 --- a/src/source/mongodb/snapshot.rs +++ b/src/source/mongodb/snapshot.rs @@ -2,17 +2,25 @@ use futures_util::StreamExt; use mongodb::bson::{doc, Timestamp}; use mongodb::Client; use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; use crate::error::{CdcError, Result}; use crate::source::SourceEvent; use super::event_converter::snapshot_doc_to_cdc_event; +use super::offset::MongoOffset; +use super::{send_event, SendOutcome}; + +/// Number of documents between snapshot progress checkpoints. +const SNAPSHOT_CHECKPOINT_INTERVAL: u64 = 1000; /// Result of a snapshot operation. pub struct SnapshotResult { /// The cluster time recorded before the snapshot started. /// Used to start change stream from the correct point. pub snapshot_time: Timestamp, + /// False if the snapshot was interrupted by a shutdown signal. + pub completed: bool, } /// Perform initial snapshot of the specified collections. @@ -20,11 +28,17 @@ pub struct SnapshotResult { /// 1. Record the current cluster time via a `ping` command. /// 2. Scan each collection with `find({})`. /// 3. Send Change + Checkpoint events for each document batch. +/// +/// Progress checkpoints are tagged `snapshot_in_progress`: snapshots are not +/// resumable mid-way, so restarting from one re-runs the whole snapshot. The +/// final checkpoint is tagged `snapshot_complete` and carries the cluster time, +/// letting a restart skip straight to the change stream. pub async fn perform_snapshot( client: &Client, database: &str, collections: &[String], sender: &mpsc::Sender, + shutdown: &CancellationToken, ) -> Result { let db = client.database(database); @@ -56,6 +70,14 @@ pub async fn perform_snapshot( }; for coll_name in &collection_names { + if shutdown.is_cancelled() { + tracing::info!("shutdown received, aborting snapshot"); + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } + tracing::info!(collection = %coll_name, "starting snapshot"); let coll = db.collection::(coll_name); @@ -65,43 +87,83 @@ pub async fn perform_snapshot( .map_err(|e| CdcError::Mongodb(format!("snapshot find on {coll_name}: {e}")))?; let mut count: u64 = 0; - while let Some(result) = cursor.next().await { + loop { + let next = tokio::select! { + _ = shutdown.cancelled() => { + tracing::info!(collection = %coll_name, count, "shutdown received, aborting snapshot"); + return Ok(SnapshotResult { snapshot_time, completed: false }); + } + next = cursor.next() => next, + }; + + let Some(result) = next else { + break; + }; + let doc = result .map_err(|e| CdcError::Mongodb(format!("snapshot cursor on {coll_name}: {e}")))?; let event = snapshot_doc_to_cdc_event(&doc, database, coll_name); - sender - .send(SourceEvent::Change(event)) - .await - .map_err(|e| CdcError::Mongodb(format!("failed to send snapshot event: {e}")))?; + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Change(event), shutdown).await? + { + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } count += 1; - // Checkpoint every 1000 documents - if count.is_multiple_of(1000) { - let offset = format!( - "{{\"snapshot\":\"{}\",\"collection\":\"{}\",\"count\":{}}}", - coll_name, coll_name, count - ); - sender - .send(SourceEvent::Checkpoint { offset }) - .await - .map_err(|e| CdcError::Mongodb(format!("failed to send checkpoint: {e}")))?; + if count.is_multiple_of(SNAPSHOT_CHECKPOINT_INTERVAL) { + let offset = MongoOffset::SnapshotInProgress { + collection: coll_name.clone(), + count, + } + .to_json()?; + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Checkpoint { offset }, shutdown).await? + { + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } } } - // Final checkpoint for this collection - let offset = format!( - "{{\"snapshot_complete\":\"{}\",\"count\":{}}}", - coll_name, count - ); - sender - .send(SourceEvent::Checkpoint { offset }) - .await - .map_err(|e| CdcError::Mongodb(format!("failed to send checkpoint: {e}")))?; + // Per-collection checkpoint: the snapshot as a whole is still in progress + // until every collection has been scanned. + let offset = MongoOffset::SnapshotInProgress { + collection: coll_name.clone(), + count, + } + .to_json()?; + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Checkpoint { offset }, shutdown).await? + { + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } tracing::info!(collection = %coll_name, count, "snapshot complete"); } - Ok(SnapshotResult { snapshot_time }) + // All collections scanned — record a resumable position so a restart skips + // the snapshot and starts the change stream at the recorded cluster time. + let offset = MongoOffset::SnapshotComplete { + cluster_time: snapshot_time.into(), + } + .to_json()?; + let completed = matches!( + send_event(sender, SourceEvent::Checkpoint { offset }, shutdown).await?, + SendOutcome::Sent + ); + + Ok(SnapshotResult { + snapshot_time, + completed, + }) } diff --git a/tests/mongodb_integration.rs b/tests/mongodb_integration.rs index 297615b..173b2ce 100644 --- a/tests/mongodb_integration.rs +++ b/tests/mongodb_integration.rs @@ -15,13 +15,15 @@ use testcontainers_modules::mongo::Mongo; use tokio::time::timeout; use tokio_util::sync::CancellationToken; -use cdcflow::config::MongodbSourceConfig; +use cdcflow::config::{MongodbSourceConfig, PostgresSinkConfig, SinkMode, SourceConnectionConfig}; use cdcflow::error::Result; use cdcflow::event::{CdcEvent, ChangeOp, ColumnValue}; use cdcflow::offset::memory::MemoryOffsetStore; use cdcflow::offset::OffsetStore; use cdcflow::pipeline::Pipeline; +use cdcflow::sink::postgres::PostgresSink; use cdcflow::sink::Sink; +use cdcflow::source::mongodb::offset::{ClusterTime, MongoOffset}; use cdcflow::source::mongodb::MongodbSource; /// A sink that collects events into a shared Vec for assertions. @@ -109,7 +111,7 @@ async fn start_mongodb() -> (ContainerAsync, String, u16) { for _ in 0..30 { tokio::time::sleep(Duration::from_secs(1)).await; if let Ok(status) = admin_db.run_command(doc! { "replSetGetStatus": 1 }).await { - if let Some(members) = status.get_array("members").ok() { + if let Ok(members) = status.get_array("members") { for member in members { if let Some(doc) = member.as_document() { if doc.get_str("stateStr").ok() == Some("PRIMARY") { @@ -536,3 +538,643 @@ async fn test_mongodb_resume_from_offset() { let _ = timeout(Duration::from_secs(5), handle).await; } } + +/// Poll `check` until it returns true or the timeout expires. +async fn wait_until(secs: u64, label: &str, mut check: F) +where + F: FnMut() -> bool, +{ + let result = timeout(Duration::from_secs(secs), async { + loop { + if check() { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await; + assert!(result.is_ok(), "timed out waiting for {label}"); +} + +/// Update and delete on a collection **without** `changeStreamPreAndPostImages` +/// (the MongoDB default) must still carry the `_id` primary key in `old`, +/// otherwise sinks cannot target the affected row. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_update_and_delete_without_pre_images() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("no_preimage_db"); + // Deliberately no collMod: pre-images stay disabled. + db.create_collection("items") + .await + .expect("failed to create collection"); + let coll = db.collection::("items"); + coll.insert_one(doc! { "_id": "item1", "name": "Widget", "price": 10.0 }) + .await + .expect("failed to insert"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "no_preimage_db".into(), + collections: vec!["items".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "snapshot", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Snapshot) + }) + .await; + } + + coll.update_one(doc! { "_id": "item1" }, doc! { "$set": { "price": 15.0 } }) + .await + .expect("failed to update"); + + { + let events = events.clone(); + wait_until(20, "update event", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Update) + }) + .await; + } + + { + let evts = events.lock().unwrap(); + let update = evts.iter().find(|e| e.op == ChangeOp::Update).unwrap(); + let old = update + .old + .as_ref() + .expect("update must carry an old row built from documentKey"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("item1".into()))); + } + + coll.delete_one(doc! { "_id": "item1" }) + .await + .expect("failed to delete"); + + { + let events = events.clone(); + wait_until(20, "delete event", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Delete) + }) + .await; + } + + { + let evts = events.lock().unwrap(); + let delete = evts.iter().find(|e| e.op == ChangeOp::Delete).unwrap(); + let old = delete + .old + .as_ref() + .expect("delete must carry an old row built from documentKey"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("item1".into()))); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// Read the deployment's current cluster time, the value a completed snapshot +/// records in its offset. +async fn current_cluster_time(client: &mongodb::Client, database: &str) -> ClusterTime { + let ping = client + .database(database) + .run_command(doc! { "ping": 1 }) + .await + .expect("ping failed"); + let ts = ping + .get("operationTime") + .and_then(|v| v.as_timestamp()) + .expect("ping response missing operationTime"); + ClusterTime::from(ts) +} + +/// A stored `snapshot_complete` offset must resume the change stream from the +/// recorded cluster time — not be misread as a resume token, and not re-snapshot. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_resumes_from_snapshot_complete_offset() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("snapshot_only_db"); + db.create_collection("docs") + .await + .expect("failed to create collection"); + let coll = db.collection::("docs"); + coll.insert_one(doc! { "name": "BeforeOffset" }) + .await + .expect("failed to insert"); + + // Exactly what the source persists once a snapshot finishes with no change + // events behind it. + let cluster_time = current_cluster_time(&client, "snapshot_only_db").await; + let offset_store = MemoryOffsetStore::with_offset( + MongoOffset::SnapshotComplete { cluster_time } + .to_json() + .unwrap(), + ); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "snapshot_only_db".into(), + collections: vec!["docs".into()], + }; + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Written after the recorded cluster time, so it must arrive on the stream. + tokio::time::sleep(Duration::from_secs(2)).await; + coll.insert_one(doc! { "name": "AfterOffset" }) + .await + .expect("failed to insert"); + + { + let events = events.clone(); + wait_until(30, "AfterOffset insert", move || { + events.lock().unwrap().iter().any(|e| { + e.op == ChangeOp::Insert + && e.new + .as_ref() + .and_then(|r| r.get("name")) + .map(|v| matches!(v, ColumnValue::Text(s) if s == "AfterOffset")) + .unwrap_or(false) + }) + }) + .await; + } + + { + let evts = events.lock().unwrap(); + assert_eq!( + evts.iter().filter(|e| e.op == ChangeOp::Snapshot).count(), + 0, + "a completed snapshot must not be re-run on restart" + ); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// A stored `snapshot_in_progress` offset means the previous snapshot was cut +/// short. Snapshots are not resumable mid-way, so the whole snapshot re-runs. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_reruns_interrupted_snapshot() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("interrupted_db"); + db.create_collection("docs") + .await + .expect("failed to create collection"); + db.collection::("docs") + .insert_many(vec![doc! { "name": "A" }, doc! { "name": "B" }]) + .await + .expect("failed to insert"); + + let offset_store = MemoryOffsetStore::with_offset( + MongoOffset::SnapshotInProgress { + collection: "docs".into(), + count: 1, + } + .to_json() + .unwrap(), + ); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "interrupted_db".into(), + collections: vec!["docs".into()], + }; + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "re-run snapshot of both documents", move || { + events + .lock() + .unwrap() + .iter() + .filter(|e| e.op == ChangeOp::Snapshot) + .count() + >= 2 + }) + .await; + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// An offset the source cannot interpret must fail loudly instead of being +/// handed to the server as a bogus resume token. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_rejects_unrecognized_offset() { + let (_container, host, port) = start_mongodb().await; + + let offset_store = + MemoryOffsetStore::with_offset(r#"{"snapshot_complete":"docs","count":42}"#.to_string()); + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "whatever_db".into(), + collections: vec![], + }; + let (sink, _events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let result = timeout(Duration::from_secs(30), pipeline.run(shutdown)) + .await + .expect("pipeline should fail fast on an unreadable offset"); + + let err = result.expect_err("an unreadable offset must be an error"); + assert!( + err.to_string().contains("failed to parse stored offset"), + "unexpected error: {err}" + ); +} + +/// An empty `collections` list snapshots and streams every collection in the database. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_all_collections_when_filter_empty() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("all_colls_db"); + for name in ["alpha", "beta"] { + db.create_collection(name) + .await + .expect("failed to create collection"); + db.collection::(name) + .insert_one(doc! { "name": name }) + .await + .expect("failed to insert"); + } + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "all_colls_db".into(), + collections: vec![], // all collections + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "snapshot of both collections", move || { + let evts = events.lock().unwrap(); + let tables: std::collections::HashSet = evts + .iter() + .filter(|e| e.op == ChangeOp::Snapshot) + .map(|e| e.table.name.clone()) + .collect(); + tables.contains("alpha") && tables.contains("beta") + }) + .await; + } + + // A change in either collection must stream through too. + db.collection::("beta") + .insert_one(doc! { "name": "beta2" }) + .await + .expect("failed to insert"); + + { + let events = events.clone(); + wait_until(20, "streamed insert on beta", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Insert && e.table.name == "beta") + }) + .await; + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// A non-empty `collections` list must exclude changes from other collections. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_collection_filter_excludes_others() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("filter_db"); + for name in ["watched", "ignored"] { + db.create_collection(name) + .await + .expect("failed to create collection"); + } + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "filter_db".into(), + collections: vec!["watched".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Give the change stream time to open before writing. + tokio::time::sleep(Duration::from_secs(3)).await; + + db.collection::("ignored") + .insert_one(doc! { "name": "nope" }) + .await + .expect("failed to insert"); + db.collection::("watched") + .insert_one(doc! { "name": "yes" }) + .await + .expect("failed to insert"); + + { + let events = events.clone(); + wait_until(20, "insert on watched collection", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Insert && e.table.name == "watched") + }) + .await; + } + + { + let evts = events.lock().unwrap(); + assert!( + evts.iter().all(|e| e.table.name != "ignored"), + "events from unwatched collections must be filtered out" + ); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// A shutdown during the snapshot must stop it promptly instead of scanning +/// every remaining document first. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_shutdown_during_snapshot() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("big_snapshot_db"); + let coll = db.collection::("many"); + let docs: Vec = (0..20_000) + .map(|i| doc! { "i": i, "payload": "x".repeat(200) }) + .collect(); + coll.insert_many(docs).await.expect("failed to seed docs"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "big_snapshot_db".into(), + collections: vec!["many".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait until the snapshot is clearly underway, then cancel mid-scan. + { + let events = events.clone(); + wait_until(30, "snapshot to start", move || { + !events.lock().unwrap().is_empty() + }) + .await; + } + shutdown.cancel(); + + let result = timeout(Duration::from_secs(10), handle) + .await + .expect("pipeline did not stop within 10s of cancellation"); + result + .expect("pipeline task panicked") + .expect("pipeline returned an error"); + + let seen = events.lock().unwrap().len(); + assert!( + seen < 20_000, + "snapshot should have been interrupted, but emitted all {seen} documents" + ); +} + +/// End-to-end replication into PostgreSQL with pre-images disabled: inserts, +/// updates and deletes must all land on the correct target row. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_to_postgres_replication() { + use testcontainers_modules::postgres::Postgres; + use tokio_postgres::NoTls; + + let (_mongo, mongo_host, mongo_port) = start_mongodb().await; + let pg_container = Postgres::default() + .with_tag("18-alpine") + .start() + .await + .expect("failed to start postgres container"); + let pg_port = pg_container + .get_host_port_ipv4(5432) + .await + .expect("failed to get pg port"); + let pg_host = pg_container + .get_host() + .await + .expect("failed to get pg host") + .to_string(); + + let mongo_client = get_client(&mongo_host, mongo_port).await; + let db = mongo_client.database("repl_db"); + db.create_collection("users") + .await + .expect("failed to create collection"); + let coll = db.collection::("users"); + coll.insert_many(vec![ + doc! { "_id": "u1", "name": "Alice", "age": 30 }, + doc! { "_id": "u2", "name": "Bob", "age": 25 }, + ]) + .await + .expect("failed to seed docs"); + + let mongo_conn = connection_string(&mongo_host, mongo_port); + let sink = PostgresSink::new( + PostgresSinkConfig { + host: pg_host.clone(), + port: pg_port, + user: "postgres".into(), + password: Some("postgres".into()), + database: "postgres".into(), + schema: "public".into(), + table_prefix: "".into(), + }, + SinkMode::Replication, + Some(SourceConnectionConfig::Mongodb { + url: mongo_conn.clone(), + }), + ) + .await + .expect("failed to create postgres sink"); + + let config = MongodbSourceConfig { + connection_string: mongo_conn, + database: "repl_db".into(), + collections: vec!["users".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Verification client. + let (pg, connection) = tokio_postgres::connect( + &format!( + "host={} port={} user=postgres password=postgres dbname=postgres", + pg_host, pg_port + ), + NoTls, + ) + .await + .expect("failed to connect to postgres"); + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("pg connection error: {e}"); + } + }); + + async fn count_users(pg: &tokio_postgres::Client) -> i64 { + pg.query_one("SELECT count(*)::bigint FROM public.users", &[]) + .await + .map(|row| row.get::<_, i64>(0)) + .unwrap_or(-1) + } + + // Snapshot replicated. + timeout(Duration::from_secs(60), async { + loop { + if count_users(&pg).await == 2 { + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .expect("timed out waiting for snapshot rows in postgres"); + + // Update (no pre-image available) must update the existing row, not add one. + coll.update_one(doc! { "_id": "u1" }, doc! { "$set": { "age": 31 } }) + .await + .expect("failed to update"); + + timeout(Duration::from_secs(60), async { + loop { + let row = pg + .query_opt("SELECT age FROM public.users WHERE _id = 'u1'", &[]) + .await + .expect("query failed"); + if let Some(row) = row { + let age: i64 = match row.try_get::<_, i64>(0) { + Ok(v) => v, + Err(_) => row.get::<_, i32>(0) as i64, + }; + if age == 31 && count_users(&pg).await == 2 { + return; + } + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .expect("timed out waiting for the update to replicate"); + + // Delete (no pre-image available) must remove the row by its `_id`. + coll.delete_one(doc! { "_id": "u2" }) + .await + .expect("failed to delete"); + + timeout(Duration::from_secs(60), async { + loop { + if count_users(&pg).await == 1 { + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .expect("timed out waiting for the delete to replicate"); + + let remaining: String = pg + .query_one("SELECT _id FROM public.users", &[]) + .await + .expect("query failed") + .get(0); + assert_eq!(remaining, "u1", "the wrong row was deleted"); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(10), handle).await; +} From 83a57cc2c887d2c9b097ea050fc19c1098cb7e1d Mon Sep 17 00:00:00 2001 From: Manfred Lee Date: Mon, 27 Jul 2026 11:17:09 -0700 Subject: [PATCH 5/6] Harden MongoDB schema discovery and change stream resumption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schema discovery sampled only the oldest 100 documents of a collection, and callers treat that sample as the authoritative source schema. A field added later in the collection's life was therefore invisible: it was typed as text, skipped by Iceberg schema evolution, or — worst — confirmed as "dropped" and removed from the replication target along with its data. Sampling now takes half the budget from each end of the collection, and drop confirmation is disabled entirely for schemaless sources, where a field's absence from a sample proves nothing. Change stream positions could also become unresumable. The resume token of an `invalidate` event was stored even though the server refuses it in `resumeAfter`, and an idle or fully filtered stream never advanced the offset at all, so it eventually aged out of the oplog. Invalidate tokens are no longer stored, a drained batch falls back to the stream's own resume token, and a start position that has fallen out of the oplog now triggers a fresh snapshot instead of failing identically on every restart. Also: - map MongoDB `int` to a 64-bit column: every BSON integer arrives as i64, so a 32-bit target truncated in Iceberg and failed the insert in Postgres - cache mongodb::Client per connection string, since discovery runs per write batch in replication mode and each call re-ran topology discovery - fail fast on a configured collection that does not exist, instead of running indefinitely and replicating nothing Tests: unit coverage for drop-confirmation gating and lost-resume-point detection; integration tests for unknown collections, offset advancement while the watched collection is idle, invalidate tokens staying out of the offset store, and a sparse MongoDB field not dropping its target column. --- src/schema/discovery.rs | 83 +++++++---- src/schema/evolution.rs | 32 +++++ src/schema/type_mapping.rs | 8 +- src/sink/iceberg/mod.rs | 10 ++ src/sink/postgres/mod.rs | 6 +- src/source/mongodb/bson_mapping.rs | 21 +-- src/source/mongodb/change_stream.rs | 121 +++++++++++----- src/source/mongodb/event_converter.rs | 21 +-- src/source/mongodb/mod.rs | 94 +++++++++++-- src/source/mongodb/offset.rs | 20 +-- src/source/mongodb/snapshot.rs | 23 +-- tests/mongodb_integration.rs | 194 ++++++++++++++++++++++++-- tests/postgres_sink_integration.rs | 78 +++++++++++ 13 files changed, 559 insertions(+), 152 deletions(-) diff --git a/src/schema/discovery.rs b/src/schema/discovery.rs index 90754fd..9c3208c 100644 --- a/src/schema/discovery.rs +++ b/src/schema/discovery.rs @@ -346,9 +346,65 @@ async fn fetch_column_subset_mysql( Ok(result) } -/// Number of documents sampled to infer a MongoDB collection's schema. const SCHEMA_SAMPLE_SIZE: i64 = 100; +/// Cached clients keyed by connection string — discovery runs per write batch, +/// and building a `mongodb::Client` re-runs topology discovery each time. +static MONGO_CLIENTS: std::sync::OnceLock< + tokio::sync::Mutex>, +> = std::sync::OnceLock::new(); + +async fn mongodb_client(url: &str) -> Result { + let mut clients = MONGO_CLIENTS + .get_or_init(|| tokio::sync::Mutex::new(std::collections::HashMap::new())) + .lock() + .await; + + if let Some(client) = clients.get(url) { + return Ok(client.clone()); + } + + let client = mongodb::Client::with_uri_str(url) + .await + .map_err(|e| CdcError::Schema(format!("mongodb connect: {e}")))?; + clients.insert(url.to_string(), client.clone()); + Ok(client) +} + +/// Sample documents from both ends of a collection and merge their field types. +/// +/// Callers treat the result as the current source schema, so both ends matter: +/// the newest documents carry fields added later, the oldest carry fields only +/// they still have. Sorted by `_id` to keep the sample stable across runs. +async fn sample_field_types( + coll: &mongodb::Collection, +) -> Result> { + use futures_util::StreamExt; + use mongodb::bson::doc; + + let mut field_types: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for order in [1, -1] { + let mut cursor = coll + .find(doc! {}) + .sort(doc! { "_id": order }) + .limit(SCHEMA_SAMPLE_SIZE / 2) + .await + .map_err(|e| CdcError::Schema(format!("mongodb find: {e}")))?; + + while let Some(result) = cursor.next().await { + let doc = result.map_err(|e| CdcError::Schema(format!("mongodb cursor: {e}")))?; + crate::source::mongodb::bson_mapping::merge_document_field_types( + &mut field_types, + &doc, + ); + } + } + + Ok(field_types) +} + /// Fetch columns by sampling documents from a MongoDB collection. /// /// Since MongoDB is schemaless, we sample up to `SCHEMA_SAMPLE_SIZE` documents @@ -360,32 +416,11 @@ async fn fetch_columns_mongodb( database: &str, collection: &str, ) -> Result> { - use mongodb::bson::doc; - - let client = mongodb::Client::with_uri_str(url) - .await - .map_err(|e| CdcError::Schema(format!("mongodb connect: {e}")))?; - + let client = mongodb_client(url).await?; let db = client.database(database); let coll = db.collection::(collection); - // Sort by `_id` so repeated runs sample the same documents and infer a - // stable schema instead of whatever the server happens to return first. - let mut cursor = coll - .find(doc! {}) - .sort(doc! { "_id": 1 }) - .limit(SCHEMA_SAMPLE_SIZE) - .await - .map_err(|e| CdcError::Schema(format!("mongodb find: {e}")))?; - - let mut field_types: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - - use futures_util::StreamExt; - while let Some(result) = cursor.next().await { - let doc = result.map_err(|e| CdcError::Schema(format!("mongodb cursor: {e}")))?; - crate::source::mongodb::bson_mapping::merge_document_field_types(&mut field_types, &doc); - } + let field_types = sample_field_types(&coll).await?; let dialect = SourceDialect::Mongodb; let columns: Vec = field_types diff --git a/src/schema/evolution.rs b/src/schema/evolution.rs index e989ea0..8d905a7 100644 --- a/src/schema/evolution.rs +++ b/src/schema/evolution.rs @@ -1,5 +1,6 @@ use std::collections::HashSet; +use crate::config::SourceConnectionConfig; use crate::event::{CdcEvent, ChangeOp}; /// Detect column names present in events but missing from the known column list. @@ -30,6 +31,14 @@ pub fn detect_new_columns(known_columns: &[String], events: &[&CdcEvent]) -> Vec new_cols } +/// Whether a column's absence from the source can be treated as a drop. +/// +/// False for schemaless sources: MongoDB discovery only samples documents, so +/// absence there would drop live columns and their replicated data. +pub fn source_supports_column_drops(source_conn: &SourceConnectionConfig) -> bool { + !matches!(source_conn, SourceConnectionConfig::Mongodb { .. }) +} + /// Detect target columns that no longer exist in the source. /// /// Returns column names present in `target_columns` but absent from `source_columns`. @@ -50,6 +59,29 @@ mod tests { use crate::event::{ColumnValue, Lsn, TableId}; use std::collections::BTreeMap; + #[test] + fn test_schemaless_sources_never_confirm_column_drops() { + assert!(!source_supports_column_drops( + &SourceConnectionConfig::Mongodb { + url: "mongodb://localhost:27017".into() + } + )); + } + + #[test] + fn test_relational_sources_confirm_column_drops() { + assert!(source_supports_column_drops( + &SourceConnectionConfig::Postgres { + url: "postgres://localhost/db".into() + } + )); + assert!(source_supports_column_drops( + &SourceConnectionConfig::Mysql { + url: "mysql://localhost/db".into() + } + )); + } + fn make_table_id() -> TableId { TableId { schema: "public".into(), diff --git a/src/schema/type_mapping.rs b/src/schema/type_mapping.rs index da6b815..2cffd65 100644 --- a/src/schema/type_mapping.rs +++ b/src/schema/type_mapping.rs @@ -147,8 +147,8 @@ fn parse_mysql_type(dt: &str) -> CanonicalType { fn parse_mongodb_type(dt: &str) -> CanonicalType { match dt { "objectid" | "string" => CanonicalType::Text, - "int" => CanonicalType::Int, - "long" => CanonicalType::BigInt, + // Both arrive as ColumnValue::Int(i64); a 32-bit column would truncate. + "int" | "long" => CanonicalType::BigInt, "double" => CanonicalType::Double, "bool" => CanonicalType::Boolean, "date" => CanonicalType::TimestampTz, @@ -526,7 +526,7 @@ mod tests { let cases = vec![ ("objectid", CanonicalType::Text), ("string", CanonicalType::Text), - ("int", CanonicalType::Int), + ("int", CanonicalType::BigInt), ("long", CanonicalType::BigInt), ("double", CanonicalType::Double), ("bool", CanonicalType::Boolean), @@ -558,7 +558,7 @@ mod tests { ); assert_eq!( parse_source_type(SourceDialect::Mongodb, "Int"), - CanonicalType::Int + CanonicalType::BigInt ); } diff --git a/src/sink/iceberg/mod.rs b/src/sink/iceberg/mod.rs index 6c57e1c..02f5cbd 100644 --- a/src/sink/iceberg/mod.rs +++ b/src/sink/iceberg/mod.rs @@ -362,6 +362,16 @@ impl IcebergSink { schema_evolution::detect_dropped_columns_from_events(arrow_schema, events, false); if has_full_row_events && !missing_from_events.is_empty() { + if !crate::schema::evolution::source_supports_column_drops(&self.source_connection) { + tracing::debug!( + schema = source_schema, + table = source_table, + ?missing_from_events, + "schemaless source: keeping columns missing from events", + ); + return Ok(()); + } + tracing::debug!( schema = source_schema, table = source_table, diff --git a/src/sink/postgres/mod.rs b/src/sink/postgres/mod.rs index ca1f1e3..d1434ac 100644 --- a/src/sink/postgres/mod.rs +++ b/src/sink/postgres/mod.rs @@ -496,7 +496,11 @@ impl PostgresSink { if !missing_from_events.is_empty() { // Confirm the drop by querying the source database - if let Some(source_conn) = self.source_connection.as_ref() { + if let Some(source_conn) = self + .source_connection + .as_ref() + .filter(|c| crate::schema::evolution::source_supports_column_drops(c)) + { let source_cols = crate::schema::discovery::fetch_columns(source_conn, schema, table) .await?; diff --git a/src/source/mongodb/bson_mapping.rs b/src/source/mongodb/bson_mapping.rs index 2b8c88a..6ee5575 100644 --- a/src/source/mongodb/bson_mapping.rs +++ b/src/source/mongodb/bson_mapping.rs @@ -18,8 +18,7 @@ pub fn bson_to_column_value(bson: &Bson) -> ColumnValue { Bson::String(s) => ColumnValue::Text(s.clone()), Bson::ObjectId(oid) => ColumnValue::Text(oid.to_hex()), Bson::DateTime(dt) => { - // Convert millis to ISO 8601 via chrono. `from_timestamp_millis` handles - // negative (pre-1970) values correctly — manual secs/nanos splitting does not. + // `from_timestamp_millis` handles negative (pre-1970) values. let ts = chrono::DateTime::from_timestamp_millis(dt.timestamp_millis()) .unwrap_or_default() .to_rfc3339(); @@ -76,17 +75,8 @@ pub fn bson_type_string(bson: &Bson) -> &'static str { } } -/// Merge two observed BSON type names for the same field into a single type -/// that can hold both. -/// -/// MongoDB is schemaless, so the same field can hold different types across -/// documents. Rules: -/// - `null` carries no type information — the other type wins. -/// - Identical types stay as-is. -/// - Integer widening: `int` + `long` → `long`. -/// - Numeric widening: any int type + `double` → `double`. -/// - Anything else conflicting falls back to `string`, which maps to a text -/// column and can hold any stringified value. +/// Merge two BSON type names observed for the same field into one that holds +/// both: nulls carry no type, numerics widen, anything else falls back to text. pub fn merge_bson_types(existing: &str, incoming: &str) -> String { if existing == incoming { return existing.to_string(); @@ -101,8 +91,6 @@ pub fn merge_bson_types(existing: &str, incoming: &str) -> String { } } -/// Fold a document's top-level fields into an accumulating field → type map, -/// merging types when a field has already been seen with a different type. pub fn merge_document_field_types(field_types: &mut BTreeMap, doc: &Document) { for (key, value) in doc { let incoming = bson_type_string(value); @@ -117,9 +105,6 @@ pub fn merge_document_field_types(field_types: &mut BTreeMap, do } } -/// Infer schema from a set of documents by building a union of all top-level -/// fields. Types observed across documents are merged (see [`merge_bson_types`]), -/// so a field that is null in one document and an int in another is typed `int`. pub fn infer_schema_from_documents(docs: &[Document]) -> Vec<(String, String)> { let mut field_types: BTreeMap = BTreeMap::new(); for doc in docs { diff --git a/src/source/mongodb/change_stream.rs b/src/source/mongodb/change_stream.rs index 9ec2d4c..ffbec1b 100644 --- a/src/source/mongodb/change_stream.rs +++ b/src/source/mongodb/change_stream.rs @@ -1,7 +1,8 @@ use std::time::Duration; use mongodb::bson::{doc, Document, Timestamp}; -use mongodb::change_stream::event::ResumeToken; +use mongodb::change_stream::event::{ChangeStreamEvent, ResumeToken}; +use mongodb::change_stream::ChangeStream; use mongodb::options::FullDocumentBeforeChangeType; use mongodb::options::FullDocumentType; use mongodb::Client; @@ -15,11 +16,9 @@ use super::event_converter::{change_event_to_cdc_event, ConvertResult}; use super::offset::MongoOffset; use super::{send_event, SendOutcome}; -/// Max change events buffered before forcing an offset commit. const MAX_EVENTS_PER_COMMIT: usize = 1000; -/// How long a `getMore` waits server-side for new events before returning empty. -/// Bounds the latency of the commit that flushes a partially filled batch. +/// Server-side wait for new events; bounds partial-batch commit latency. const MAX_AWAIT_TIME: Duration = Duration::from_millis(500); /// Determines where to start the change stream from. @@ -30,26 +29,16 @@ pub enum ResumePoint { Token(ResumeToken), } -/// Run the change stream loop, sending CDC events to the pipeline. -/// -/// Supports filtering to specific collections via a `$match` pipeline stage. -/// Uses `fullDocument: "updateLookup"` to get the full document on updates, -/// and `fullDocumentBeforeChange: "whenAvailable"` to capture pre-images. -/// -/// Events are drained batch-by-batch: a `Commit` carrying the latest resume -/// token is emitted once a server batch is exhausted (or after -/// `MAX_EVENTS_PER_COMMIT` events), rather than once per document. -pub async fn run_change_stream( +/// Uses `updateLookup` for the post-image and `whenAvailable` for the +/// pre-image; `collections` becomes a `$match` stage. +pub async fn open_change_stream( client: &Client, database: &str, collections: &[String], - resume_point: ResumePoint, - sender: &mpsc::Sender, - shutdown: &CancellationToken, -) -> Result<()> { + resume_point: &ResumePoint, +) -> Result>> { let db = client.database(database); - // Build pipeline to filter collections if specified let pipeline: Vec = if collections.is_empty() { vec![] } else { @@ -67,7 +56,7 @@ pub async fn run_change_stream( .full_document_before_change(FullDocumentBeforeChangeType::WhenAvailable) .max_await_time(MAX_AWAIT_TIME); - match &resume_point { + match resume_point { ResumePoint::AfterSnapshot { snapshot_time } => { builder = builder.start_at_operation_time(*snapshot_time); } @@ -76,13 +65,31 @@ pub async fn run_change_stream( } } - let mut stream = builder + builder .await - .map_err(|e| CdcError::Mongodb(format!("failed to open change stream: {e}")))?; + .map_err(|e| CdcError::Mongodb(format!("failed to open change stream: {e}"))) +} + +/// True if the start position has aged out of the oplog — it can never become +/// valid again, so the only way forward is a fresh snapshot. +pub fn is_resume_point_lost(err: &CdcError) -> bool { + let msg = err.to_string(); + msg.contains("ChangeStreamHistoryLost") + || msg.contains("resume point may no longer be in the oplog") + || msg.contains("as the resume point may no longer be in the oplog") + || msg.contains("resume of change stream was not possible") +} +/// Commits the latest resume token once a server batch is drained (or after +/// `MAX_EVENTS_PER_COMMIT` events), not once per document. +pub async fn pump_change_stream( + stream: &mut ChangeStream>, + database: &str, + sender: &mpsc::Sender, + shutdown: &CancellationToken, +) -> Result<()> { tracing::info!(database, "change stream started"); - // Latest resume token not yet committed, and how many events it covers. let mut pending_offset: Option = None; let mut pending_events: usize = 0; @@ -93,17 +100,23 @@ pub async fn run_change_stream( commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown).await?; return Ok(()); } - // `next_if_any` performs at most one `getMore`, so an empty result - // means the current batch is drained — the point at which the driver - // documentation recommends persisting the resume token. + // At most one `getMore`: an empty result means the batch is drained. next = stream.next_if_any() => next, }; match next { Ok(Some(event)) => { - // Track the token even for skipped events so the stream position - // still advances past operations we do not forward. - let offset = MongoOffset::Token(event.id.clone()).to_json()?; + // Skipped events still advance the position — except + // `invalidate`, which the server refuses in `resumeAfter`. + let is_invalidate = matches!( + event.operation_type, + mongodb::change_stream::event::OperationType::Invalidate + ); + let offset = if is_invalidate { + None + } else { + Some(MongoOffset::Token(event.id.clone()).to_json()?) + }; match change_event_to_cdc_event(&event, database)? { ConvertResult::Event(cdc_event) => { @@ -118,7 +131,9 @@ pub async fn run_change_stream( } } - pending_offset = Some(offset); + if let Some(offset) = offset { + pending_offset = Some(offset); + } pending_events += 1; if pending_events >= MAX_EVENTS_PER_COMMIT { @@ -127,7 +142,14 @@ pub async fn run_change_stream( } } Ok(None) => { - // Batch drained: commit what we have, then check the stream is alive. + // With no event token (idle or fully filtered stream), the + // stream's own token keeps the offset from ageing out. + if pending_offset.is_none() && stream.is_alive() { + if let Some(token) = stream.resume_token() { + pending_offset = Some(MongoOffset::Token(token).to_json()?); + } + } + commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown).await?; if !stream.is_alive() { @@ -145,7 +167,6 @@ pub async fn run_change_stream( } } -/// Emit a `Commit` for the pending resume token, if any. async fn commit_pending( sender: &mpsc::Sender, pending_offset: &mut Option, @@ -158,3 +179,39 @@ async fn commit_pending( *pending_events = 0; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lost_resume_point_errors_are_recognized() { + let cases = [ + "failed to open change stream: Kind: Command failed: Error code 286 \ + (ChangeStreamHistoryLost)", + "failed to open change stream: resume of change stream was not possible, as the \ + resume point may no longer be in the oplog", + ]; + for msg in cases { + assert!( + is_resume_point_lost(&CdcError::Mongodb(msg.into())), + "expected a lost resume point for: {msg}" + ); + } + } + + #[test] + fn test_other_errors_are_not_treated_as_lost_resume_points() { + let cases = [ + "failed to open change stream: connection refused", + "change stream error: not authorized on demo to execute command", + "failed to parse stored offset", + ]; + for msg in cases { + assert!( + !is_resume_point_lost(&CdcError::Mongodb(msg.into())), + "did not expect a lost resume point for: {msg}" + ); + } + } +} diff --git a/src/source/mongodb/event_converter.rs b/src/source/mongodb/event_converter.rs index 1045321..763992f 100644 --- a/src/source/mongodb/event_converter.rs +++ b/src/source/mongodb/event_converter.rs @@ -10,20 +10,14 @@ use super::bson_mapping::document_to_row; pub enum ConvertResult { /// A CDC event to send downstream. Event(CdcEvent), - /// Skip this event (unsupported operation type, or a CRUD event whose - /// document could no longer be looked up). + /// Unsupported operation, or a CRUD event with no usable row. Skip, } /// Convert a MongoDB change stream event to a `CdcEvent`. /// -/// Returns `ConvertResult::Event` for insert/update/replace/delete, -/// `ConvertResult::Skip` for unsupported operations (drop, rename, etc.). -/// -/// `old` is populated from the pre-image when the collection has -/// `changeStreamPreAndPostImages` enabled, and otherwise falls back to -/// `documentKey` so that update/delete events always carry the `_id` primary -/// key that sinks need to target the right row. +/// Skips unsupported operations (drop, rename, etc.). `old` comes from the +/// pre-image, falling back to `documentKey` so update/delete always carry `_id`. pub fn change_event_to_cdc_event( event: &ChangeStreamEvent, database: &str, @@ -42,7 +36,7 @@ pub fn change_event_to_cdc_event( .and_then(|ns| ns.coll.as_deref()) .unwrap_or("unknown"); - // Prefer wall time (millisecond resolution) over cluster time (second resolution). + // Wall time is millisecond resolution; cluster time only seconds. let timestamp_us = event .wall_time .map(|wt| wt.timestamp_millis() * 1000) @@ -61,10 +55,8 @@ pub fn change_event_to_cdc_event( _ => None, }; - // An insert/update/replace with no full document means the document was - // removed between the change and the `updateLookup`. A later delete event - // covers it, so skip rather than emitting a row-less change that sinks - // cannot apply. + // No full document: removed before `updateLookup` resolved it. The later + // delete event covers it, so skip rather than emit a row-less change. if matches!(op, ChangeOp::Insert | ChangeOp::Update) && new.is_none() { tracing::warn!( collection, @@ -74,7 +66,6 @@ pub fn change_event_to_cdc_event( return Ok(ConvertResult::Skip); } - // A delete with neither pre-image nor documentKey has no way to identify the row. if op == ChangeOp::Delete && old.is_none() { tracing::warn!( collection, diff --git a/src/source/mongodb/mod.rs b/src/source/mongodb/mod.rs index e2d8d11..d020d1b 100644 --- a/src/source/mongodb/mod.rs +++ b/src/source/mongodb/mod.rs @@ -12,22 +12,58 @@ use crate::error::{CdcError, Result}; use crate::offset::OffsetStore; use crate::source::Source; -use self::change_stream::{run_change_stream, ResumePoint}; +use self::change_stream::{ + is_resume_point_lost, open_change_stream, pump_change_stream, ResumePoint, +}; use self::offset::MongoOffset; use self::snapshot::perform_snapshot; -/// Outcome of sending an event to the pipeline. +/// Fail fast on a configured collection that does not exist — MongoDB answers +/// reads on one with empty results, so a typo would silently replicate nothing. +async fn validate_collections( + client: &mongodb::Client, + database: &str, + collections: &[String], +) -> Result<()> { + if collections.is_empty() { + return Ok(()); + } + + let existing = client + .database(database) + .list_collection_names() + .await + .map_err(|e| CdcError::Mongodb(format!("failed to list collections: {e}")))?; + + let missing: Vec<&str> = collections + .iter() + .map(|c| c.as_str()) + .filter(|c| !existing.iter().any(|e| e == c)) + .collect(); + + if !missing.is_empty() { + return Err(CdcError::Mongodb(format!( + "configured collection(s) not found in database {database}: {}. Available: {}", + missing.join(", "), + if existing.is_empty() { + "(none)".to_string() + } else { + existing.join(", ") + } + ))); + } + + Ok(()) +} + pub(crate) enum SendOutcome { Sent, /// The pipeline stopped receiving while a shutdown was in flight. ShuttingDown, } -/// Send an event to the pipeline, treating a closed channel during shutdown as -/// a clean stop rather than an error. -/// -/// The pipeline's receive loop exits as soon as the shutdown token fires, which -/// can happen between a source's own cancellation checks and its next send. +/// Send an event to the pipeline. The receive loop exits as soon as the +/// shutdown token fires, so a closed channel then is a clean stop, not an error. pub(crate) async fn send_event( sender: &mpsc::Sender, event: crate::source::SourceEvent, @@ -74,6 +110,8 @@ impl Source for MongodbSource { .await .map_err(|e| CdcError::Mongodb(format!("failed to connect: {e}")))?; + validate_collections(&client, &self.config.database, &self.config.collections).await?; + // Phase 1: Determine start position (snapshot or resume) let saved_offset = self.offset_store.load().await?; @@ -137,14 +175,48 @@ impl Source for MongodbSource { "starting change stream" ); - run_change_stream( + let mut stream = match open_change_stream( &client, &self.config.database, &self.config.collections, - resume_point, - &sender, - &shutdown, + &resume_point, ) .await + { + Ok(stream) => stream, + // Position gone from the oplog: re-snapshot, or every restart wedges. + Err(e) if is_resume_point_lost(&e) => { + tracing::warn!( + error = %e, + "stored change stream position is no longer in the oplog; re-running the snapshot" + ); + + let result = perform_snapshot( + &client, + &self.config.database, + &self.config.collections, + &sender, + &shutdown, + ) + .await?; + + if !result.completed { + return Ok(()); + } + + open_change_stream( + &client, + &self.config.database, + &self.config.collections, + &ResumePoint::AfterSnapshot { + snapshot_time: result.snapshot_time, + }, + ) + .await? + } + Err(e) => return Err(e), + }; + + pump_change_stream(&mut stream, &self.config.database, &sender, &shutdown).await } } diff --git a/src/source/mongodb/offset.rs b/src/source/mongodb/offset.rs index ebaaa95..e6a8621 100644 --- a/src/source/mongodb/offset.rs +++ b/src/source/mongodb/offset.rs @@ -4,10 +4,8 @@ use serde::{Deserialize, Serialize}; use crate::error::{CdcError, Result}; -/// A cluster time (BSON timestamp) in a form that round-trips through plain JSON. -/// -/// `bson::Timestamp`'s own serde representation depends on the serializer, so the -/// two components are stored explicitly instead. +/// Cluster time split into its two components — `bson::Timestamp`'s own serde +/// representation depends on the serializer, so it does not round-trip as JSON. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct ClusterTime { pub time: u32, @@ -34,31 +32,25 @@ impl From for Timestamp { /// The persisted offset for a MongoDB source. /// -/// Externally tagged so every stored offset carries an unambiguous kind. A bare -/// resume token must never be stored on its own: `ResumeToken` deserializes from -/// *any* JSON object, so an untagged offset from the snapshot phase would silently -/// be accepted as a (bogus) resume token and rejected by the server on resume. +/// Tagged because `ResumeToken` deserializes from *any* JSON object: an +/// untagged snapshot offset would be read back as a bogus resume token. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum MongoOffset { - /// Snapshot is partially done. Snapshots are not resumable mid-way, so a - /// restart from this offset re-runs the whole snapshot. + /// Snapshots are not resumable mid-way — this re-runs the whole snapshot. SnapshotInProgress { collection: String, count: u64 }, - /// Snapshot finished for all collections. The change stream starts at the - /// cluster time recorded before the snapshot began. + /// All collections scanned; stream starts at the recorded cluster time. SnapshotComplete { cluster_time: ClusterTime }, /// Streaming position: an opaque change stream resume token. Token(ResumeToken), } impl MongoOffset { - /// Serialize to the JSON string persisted by the offset store. pub fn to_json(&self) -> Result { serde_json::to_string(self) .map_err(|e| CdcError::Mongodb(format!("failed to serialize offset: {e}"))) } - /// Parse an offset previously produced by [`MongoOffset::to_json`]. pub fn from_json(s: &str) -> Result { serde_json::from_str(s).map_err(|e| { CdcError::Mongodb(format!( diff --git a/src/source/mongodb/snapshot.rs b/src/source/mongodb/snapshot.rs index 122a9b9..6b87b8d 100644 --- a/src/source/mongodb/snapshot.rs +++ b/src/source/mongodb/snapshot.rs @@ -11,13 +11,10 @@ use super::event_converter::snapshot_doc_to_cdc_event; use super::offset::MongoOffset; use super::{send_event, SendOutcome}; -/// Number of documents between snapshot progress checkpoints. const SNAPSHOT_CHECKPOINT_INTERVAL: u64 = 1000; -/// Result of a snapshot operation. pub struct SnapshotResult { - /// The cluster time recorded before the snapshot started. - /// Used to start change stream from the correct point. + /// Cluster time recorded before the scan; where the change stream starts. pub snapshot_time: Timestamp, /// False if the snapshot was interrupted by a shutdown signal. pub completed: bool, @@ -25,14 +22,9 @@ pub struct SnapshotResult { /// Perform initial snapshot of the specified collections. /// -/// 1. Record the current cluster time via a `ping` command. -/// 2. Scan each collection with `find({})`. -/// 3. Send Change + Checkpoint events for each document batch. -/// -/// Progress checkpoints are tagged `snapshot_in_progress`: snapshots are not -/// resumable mid-way, so restarting from one re-runs the whole snapshot. The -/// final checkpoint is tagged `snapshot_complete` and carries the cluster time, -/// letting a restart skip straight to the change stream. +/// Records the cluster time via `ping`, then scans each collection with +/// `find({})`. Progress checkpoints are `snapshot_in_progress`; the final one +/// is `snapshot_complete` and carries the cluster time to stream from. pub async fn perform_snapshot( client: &Client, database: &str, @@ -42,7 +34,6 @@ pub async fn perform_snapshot( ) -> Result { let db = client.database(database); - // Record cluster time before snapshot let ping_result = db .run_command(doc! { "ping": 1 }) .await @@ -60,7 +51,6 @@ pub async fn perform_snapshot( "recorded cluster time for snapshot" ); - // Determine collections to snapshot let collection_names = if collections.is_empty() { db.list_collection_names() .await @@ -132,8 +122,7 @@ pub async fn perform_snapshot( } } - // Per-collection checkpoint: the snapshot as a whole is still in progress - // until every collection has been scanned. + // Still in progress until every collection has been scanned. let offset = MongoOffset::SnapshotInProgress { collection: coll_name.clone(), count, @@ -151,8 +140,6 @@ pub async fn perform_snapshot( tracing::info!(collection = %coll_name, count, "snapshot complete"); } - // All collections scanned — record a resumable position so a restart skips - // the snapshot and starts the change stream at the recorded cluster time. let offset = MongoOffset::SnapshotComplete { cluster_time: snapshot_time.into(), } diff --git a/tests/mongodb_integration.rs b/tests/mongodb_integration.rs index 173b2ce..20a3ede 100644 --- a/tests/mongodb_integration.rs +++ b/tests/mongodb_integration.rs @@ -556,9 +556,8 @@ where assert!(result.is_ok(), "timed out waiting for {label}"); } -/// Update and delete on a collection **without** `changeStreamPreAndPostImages` -/// (the MongoDB default) must still carry the `_id` primary key in `old`, -/// otherwise sinks cannot target the affected row. +/// Without pre-images (the MongoDB default), update and delete must still +/// carry `_id` in `old` so sinks can target the row. #[tokio::test] #[ignore = "requires Docker"] async fn test_mongodb_update_and_delete_without_pre_images() { @@ -658,8 +657,7 @@ async fn test_mongodb_update_and_delete_without_pre_images() { let _ = timeout(Duration::from_secs(5), handle).await; } -/// Read the deployment's current cluster time, the value a completed snapshot -/// records in its offset. +/// Current cluster time — what a completed snapshot records in its offset. async fn current_cluster_time(client: &mongodb::Client, database: &str) -> ClusterTime { let ping = client .database(database) @@ -673,8 +671,7 @@ async fn current_cluster_time(client: &mongodb::Client, database: &str) -> Clust ClusterTime::from(ts) } -/// A stored `snapshot_complete` offset must resume the change stream from the -/// recorded cluster time — not be misread as a resume token, and not re-snapshot. +/// A `snapshot_complete` offset resumes from its cluster time, no re-snapshot. #[tokio::test] #[ignore = "requires Docker"] async fn test_mongodb_resumes_from_snapshot_complete_offset() { @@ -746,8 +743,7 @@ async fn test_mongodb_resumes_from_snapshot_complete_offset() { let _ = timeout(Duration::from_secs(5), handle).await; } -/// A stored `snapshot_in_progress` offset means the previous snapshot was cut -/// short. Snapshots are not resumable mid-way, so the whole snapshot re-runs. +/// Snapshots are not resumable mid-way, so `snapshot_in_progress` re-runs it. #[tokio::test] #[ignore = "requires Docker"] async fn test_mongodb_reruns_interrupted_snapshot() { @@ -803,8 +799,7 @@ async fn test_mongodb_reruns_interrupted_snapshot() { let _ = timeout(Duration::from_secs(5), handle).await; } -/// An offset the source cannot interpret must fail loudly instead of being -/// handed to the server as a bogus resume token. +/// An uninterpretable offset must fail loudly, not become a bogus token. #[tokio::test] #[ignore = "requires Docker"] async fn test_mongodb_rejects_unrecognized_offset() { @@ -968,8 +963,7 @@ async fn test_mongodb_collection_filter_excludes_others() { let _ = timeout(Duration::from_secs(5), handle).await; } -/// A shutdown during the snapshot must stop it promptly instead of scanning -/// every remaining document first. +/// Shutdown during a snapshot must stop it promptly, mid-scan. #[tokio::test] #[ignore = "requires Docker"] async fn test_mongodb_shutdown_during_snapshot() { @@ -1022,8 +1016,7 @@ async fn test_mongodb_shutdown_during_snapshot() { ); } -/// End-to-end replication into PostgreSQL with pre-images disabled: inserts, -/// updates and deletes must all land on the correct target row. +/// End-to-end replication into PostgreSQL with pre-images disabled. #[tokio::test] #[ignore = "requires Docker"] async fn test_mongodb_to_postgres_replication() { @@ -1178,3 +1171,174 @@ async fn test_mongodb_to_postgres_replication() { shutdown.cancel(); let _ = timeout(Duration::from_secs(10), handle).await; } + +/// A missing collection must fail fast, not run forever replicating nothing. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_rejects_unknown_collection() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("typo_db"); + db.create_collection("users") + .await + .expect("failed to create collection"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "typo_db".into(), + collections: vec!["Users".into()], // wrong case + }; + + let (sink, _events) = CollectorSink::new(); + let offset_store = MemoryOffsetStore::new(); + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let result = timeout( + Duration::from_secs(30), + pipeline.run(CancellationToken::new()), + ) + .await + .expect("pipeline should fail fast on an unknown collection"); + + let err = result.expect_err("an unknown collection must be an error"); + assert!( + err.to_string().contains("not found in database typo_db"), + "unexpected error: {err}" + ); +} + +/// Writes to filtered-out collections must still advance the stored offset, +/// otherwise it ages out of the oplog while the watched collection is idle. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_offset_advances_while_watched_collection_idle() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("idle_db"); + for name in ["watched", "busy"] { + db.create_collection(name) + .await + .expect("failed to create collection"); + } + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "idle_db".into(), + collections: vec!["watched".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, _events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store.clone()); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Let the snapshot finish and the stream open. + tokio::time::sleep(Duration::from_secs(3)).await; + let baseline = offset_store.load().await.unwrap(); + + // Traffic only on the collection that is filtered out. + for i in 0..5 { + db.collection::("busy") + .insert_one(doc! { "i": i }) + .await + .expect("failed to insert"); + tokio::time::sleep(Duration::from_millis(200)).await; + } + + timeout(Duration::from_secs(30), async { + loop { + let current = offset_store.load().await.unwrap(); + if current.is_some() && current != baseline { + return; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + }) + .await + .expect("timed out waiting for the offset to advance past the filtered writes"); + + let advanced = offset_store.load().await.unwrap().unwrap(); + assert!( + matches!(MongoOffset::from_json(&advanced), Ok(MongoOffset::Token(_))), + "expected a resume token offset, got {advanced}" + ); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(10), handle).await; +} + +/// Dropping the database ends the stream with an `invalidate` event, whose +/// token must never be stored: the server refuses it in `resumeAfter`. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_invalidate_token_is_not_persisted() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("dropped_db"); + db.create_collection("docs") + .await + .expect("failed to create collection"); + let coll = db.collection::("docs"); + coll.insert_one(doc! { "name": "First" }) + .await + .expect("failed to insert"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "dropped_db".into(), + collections: vec![], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store.clone()); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "snapshot", move || !events.lock().unwrap().is_empty()).await; + } + tokio::time::sleep(Duration::from_secs(2)).await; + + db.drop().await.expect("failed to drop database"); + + let result = timeout(Duration::from_secs(30), handle) + .await + .expect("pipeline should stop after the stream is invalidated") + .expect("pipeline task panicked"); + assert!( + result.is_err(), + "a server-closed change stream must surface as an error" + ); + shutdown.cancel(); + + // Whatever was stored must still be usable as a resume point. + let saved = offset_store + .load() + .await + .unwrap() + .expect("an offset should have been stored"); + match MongoOffset::from_json(&saved).expect("stored offset must parse") { + MongoOffset::Token(token) => { + client + .database("dropped_db") + .watch() + .resume_after(token) + .await + .expect("stored resume token must be accepted by the server"); + } + MongoOffset::SnapshotComplete { .. } | MongoOffset::SnapshotInProgress { .. } => {} + } +} diff --git a/tests/postgres_sink_integration.rs b/tests/postgres_sink_integration.rs index a2b5de8..7d94820 100644 --- a/tests/postgres_sink_integration.rs +++ b/tests/postgres_sink_integration.rs @@ -1128,3 +1128,81 @@ async fn test_replication_mode_mongodb_source_upsert_delete() { .get(0); assert_eq!(count, 0); } + +/// A field missing from a batch of MongoDB documents is not a dropped column — +/// the target column and its data must survive. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_replication_mode_mongodb_sparse_field_does_not_drop_column() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"items\" ( + \"_id\" TEXT NOT NULL, + \"name\" TEXT, + \"nickname\" TEXT, + PRIMARY KEY (\"_id\") + )", + &[], + ) + .await + .unwrap(); + + let mut sink = PostgresSink::new( + make_sink_config(&host, port), + SinkMode::Replication, + Some(SourceConnectionConfig::Mongodb { + // Never dialled: the drop path short-circuits for schemaless sources. + url: "mongodb://127.0.0.1:1/?connectTimeoutMS=100".into(), + }), + ) + .await + .unwrap(); + + // A full-row event that omits `nickname`, as such documents routinely do. + let event = CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text("item1".into())), + ("name".to_string(), ColumnValue::Text("Widget".into())), + ] + .into_iter() + .collect(), + ), + old: None, + primary_key_columns: vec!["_id".into()], + }; + + sink.write_batch(&[event]).await.unwrap(); + sink.flush().await.unwrap(); + + let exists: bool = client + .query_one( + "SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'items' + AND column_name = 'nickname' + )", + &[], + ) + .await + .unwrap() + .get(0); + + assert!( + exists, + "a field missing from one MongoDB batch must not drop the target column" + ); +} From 5104df0fef903b57fd17d46a0fbdadcfc05afcf6 Mon Sep 17 00:00:00 2001 From: Manfred Lee Date: Tue, 28 Jul 2026 15:16:40 -0700 Subject: [PATCH 6/6] Fix sink correctness bugs found while testing the MongoDB source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Postgres sink: - Escape embedded double quotes in identifiers. MongoDB field names may contain `"`, which terminated the quoted identifier and let the rest of the name run as SQL — a crafted field name could execute arbitrary DDL. - Prefix bytes parameters with `\x`. Params are sent as TEXT and cast server-side, so bare hex was read as the escape format and stored literally, double-encoding every bytea value. - Surface the wrapped DbError: tokio-postgres renders server failures as the bare string "db error", hiding the actual cause. Iceberg sink: - Keep only the last event per primary key in a replication batch. Equality deletes never apply to data files added by the same commit, so two versions of a key in one batch both survived as duplicate rows. - Reject values that cannot be converted to the target column type instead of writing NULL. A schemaless source changing a field's type silently dropped the value. Also replaces a truncating `as i32` cast and a boolean parse that treated any unrecognised string as false. MongoDB source: - Explain BSONObjectTooLarge failures. An update to a large document with pre-images enabled produces an event over the 16 MB limit that replays on every restart; the driver's text gives no hint at the way out. --- src/error.rs | 20 +- src/sink/iceberg/record_batch.rs | 390 +++++++++++++++++++--------- src/sink/postgres/mod.rs | 55 ++-- src/sink/postgres/schema.rs | 45 +++- src/source/mongodb/change_stream.rs | 37 ++- tests/postgres_sink_integration.rs | 54 ++++ 6 files changed, 452 insertions(+), 149 deletions(-) diff --git a/src/error.rs b/src/error.rs index 932fed9..b823cb3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,7 +2,7 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum CdcError { - #[error("postgres error: {0}")] + #[error("postgres error: {}", format_postgres_error(.0))] Postgres(#[from] tokio_postgres::Error), #[error("protocol error: {0}")] @@ -48,4 +48,22 @@ pub enum CdcError { Schema(String), } +/// `tokio_postgres::Error` renders server failures as the bare string +/// "db error" — the useful text only lives in the wrapped `DbError`. +fn format_postgres_error(err: &tokio_postgres::Error) -> String { + match err.as_db_error() { + Some(db) => { + let mut msg = format!("{}: {}", db.severity(), db.message()); + if let Some(detail) = db.detail() { + msg.push_str(&format!(" (detail: {detail})")); + } + if let Some(column) = db.column() { + msg.push_str(&format!(" (column: {column})")); + } + msg + } + None => err.to_string(), + } +} + pub type Result = std::result::Result; diff --git a/src/sink/iceberg/record_batch.rs b/src/sink/iceberg/record_batch.rs index dae1d2e..3877c19 100644 --- a/src/sink/iceberg/record_batch.rs +++ b/src/sink/iceberg/record_batch.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::Arc; use arrow_array::{ @@ -74,6 +75,35 @@ pub fn events_to_record_batch( .map_err(|e| CdcError::Iceberg(format!("record batch: {e}"))) } +/// A source value that is present but cannot be represented in the target +/// column — a schemaless source changed the field's type after the table was +/// created. Writing NULL would drop the value without a trace. +fn type_mismatch(col_name: &str, data_type: &DataType, value: &ColumnValue) -> CdcError { + CdcError::Iceberg(format!( + "column {col_name} is {data_type} but the source sent {} ({}); \ + the source field type changed and the value cannot be converted", + crate::schema::value::column_value_to_string(Some(value)).unwrap_or_default(), + crate::schema::value::column_value_type_name(value), + )) +} + +fn convert_values( + col_name: &str, + data_type: &DataType, + values: &[Option<&ColumnValue>], + convert: impl Fn(&ColumnValue) -> Option, +) -> Result>> { + values + .iter() + .map(|v| match v { + None | Some(ColumnValue::Null) | Some(ColumnValue::UnchangedToast) => Ok(None), + Some(value) => convert(value) + .map(Some) + .ok_or_else(|| type_mismatch(col_name, data_type, value)), + }) + .collect() +} + /// Build an Arrow array for a source column from the extracted rows. fn build_source_column( col_name: &str, @@ -88,73 +118,58 @@ fn build_source_column( match data_type { DataType::Int32 => { - let arr: Int32Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Int(n)) => Some(*n as i32), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Int(n) => i32::try_from(*n).ok(), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Int32Array::from(vals))) } DataType::Int64 => { - let arr: Int64Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Int(n)) => Some(*n), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Int(n) => Some(*n), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Int64Array::from(vals))) } DataType::Float32 => { - let arr: Float32Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Float(f)) => Some(f.into_inner() as f32), - Some(ColumnValue::Int(n)) => Some(*n as f32), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Float(f) => Some(f.into_inner() as f32), + ColumnValue::Int(n) => Some(*n as f32), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Float32Array::from(vals))) } DataType::Float64 => { - let arr: Float64Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Float(f)) => Some(f.into_inner()), - Some(ColumnValue::Int(n)) => Some(*n as f64), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Float(f) => Some(f.into_inner()), + ColumnValue::Int(n) => Some(*n as f64), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Float64Array::from(vals))) } DataType::Boolean => { - let arr: BooleanArray = values - .iter() - .map(|v| match v { - Some(ColumnValue::Bool(b)) => Some(*b), - Some(ColumnValue::Int(n)) => Some(*n != 0), - Some(ColumnValue::Text(s)) => Some(s == "t" || s == "true" || s == "1"), + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Bool(b) => Some(*b), + ColumnValue::Int(n) => Some(*n != 0), + ColumnValue::Text(s) => match s.as_str() { + "t" | "true" | "TRUE" | "1" => Some(true), + "f" | "false" | "FALSE" | "0" => Some(false), _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + }, + _ => None, + })?; + Ok(Arc::new(BooleanArray::from(vals))) } DataType::Timestamp(TimeUnit::Microsecond, tz) => { - let arr: TimestampMicrosecondArray = values - .iter() - .map(|v| match v { - Some(ColumnValue::Timestamp(s)) | Some(ColumnValue::Text(s)) => { - parse_timestamp_us(s) - } - _ => None, - }) - .collect(); + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Timestamp(s) | ColumnValue::Text(s) => parse_timestamp_us(s), + _ => None, + })?; + let arr = TimestampMicrosecondArray::from(vals); let arr = match tz { Some(tz) => arr.with_timezone(tz.as_ref()), None => arr, @@ -162,78 +177,66 @@ fn build_source_column( Ok(Arc::new(arr)) } DataType::Date32 => { - let arr: Date32Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Date(s)) | Some(ColumnValue::Text(s)) => { - chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") - .ok() - .map(|d| { - (d - chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()) - .num_days() as i32 - }) - } - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Date(s) | ColumnValue::Text(s) => { + chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") + .ok() + .map(|d| { + (d - chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()).num_days() + as i32 + }) + } + _ => None, + })?; + Ok(Arc::new(Date32Array::from(vals))) } DataType::Decimal128(precision, scale) => { let scale_i32 = *scale as i32; - let arr: Vec> = values - .iter() - .map(|v| match v { - Some(ColumnValue::Text(s)) => parse_decimal_i128(s, scale_i32), - Some(ColumnValue::Float(f)) => { - let multiplier = 10_f64.powi(scale_i32); - Some((f.into_inner() * multiplier).round() as i128) - } - Some(ColumnValue::Int(n)) => { - let multiplier = 10_i128.pow(scale_i32 as u32); - Some(*n as i128 * multiplier) - } - _ => None, - }) - .collect(); - let arr = Decimal128Array::from(arr) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Text(s) => parse_decimal_i128(s, scale_i32), + ColumnValue::Float(f) => { + let multiplier = 10_f64.powi(scale_i32); + Some((f.into_inner() * multiplier).round() as i128) + } + ColumnValue::Int(n) => { + let multiplier = 10_i128.pow(scale_i32 as u32); + Some(*n as i128 * multiplier) + } + _ => None, + })?; + let arr = Decimal128Array::from(vals) .with_precision_and_scale(*precision, *scale) .map_err(|e| CdcError::Iceberg(format!("decimal array: {e}")))?; Ok(Arc::new(arr)) } DataType::Time64(TimeUnit::Microsecond) => { - let arr: Time64MicrosecondArray = values - .iter() - .map(|v| match v { - Some(ColumnValue::Time(s)) | Some(ColumnValue::Text(s)) => parse_time_us(s), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Time(s) | ColumnValue::Text(s) => parse_time_us(s), + _ => None, + })?; + Ok(Arc::new(Time64MicrosecondArray::from(vals))) } DataType::LargeBinary => { - let arr: LargeBinaryArray = values + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Bytes(b) => Some(b.clone()), + _ => None, + })?; + let arr: LargeBinaryArray = vals .iter() - .map(|v| match v { - Some(ColumnValue::Bytes(b)) => Some(b.as_slice()), - _ => None, - }) + .map(|v| v.as_ref().map(|b| b.as_slice())) .collect(); Ok(Arc::new(arr)) } DataType::FixedSizeBinary(16) => { - // UUID: parse from string or pass through bytes - let byte_values: Vec> = values - .iter() - .map(|v| match v { - Some(ColumnValue::Text(s)) => parse_uuid_bytes(s), - Some(ColumnValue::Bytes(b)) if b.len() == 16 => { - let mut arr = [0u8; 16]; - arr.copy_from_slice(b); - Some(arr) - } - _ => None, - }) - .collect(); + let byte_values = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Text(s) => parse_uuid_bytes(s), + ColumnValue::Bytes(b) if b.len() == 16 => { + let mut arr = [0u8; 16]; + arr.copy_from_slice(b); + Some(arr) + } + _ => None, + })?; let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size( byte_values.iter().map(|v| v.as_ref().map(|b| b.as_slice())), 16, @@ -332,23 +335,53 @@ pub fn events_to_flattened_cdc_batch( .map_err(|e| CdcError::Iceberg(format!("flattened CDC record batch: {e}"))) } +/// Identity of a row within a batch, or None when the event carries no usable PK. +fn row_key(event: &CdcEvent) -> Option>> { + let row = event.new.as_ref()?; + if event.primary_key_columns.is_empty() { + return None; + } + Some( + event + .primary_key_columns + .iter() + .map(|c| crate::schema::value::column_value_to_string(row.get(c))) + .collect(), + ) +} + /// Convert CDC events into a data RecordBatch for replication mode (source columns only). /// /// Filters to Insert/Update/Snapshot events and uses `event.new` for row data. /// Returns None if no matching events exist. +/// +/// Only the last event per primary key survives: equality deletes never apply to +/// data files added by the same commit, so two versions of a key in one batch +/// would both remain in the table. pub fn events_to_replication_data_batch( events: &[CdcEvent], arrow_schema: &ArrowSchema, ) -> Result> { - let data_events: Vec<&CdcEvent> = events - .iter() - .filter(|e| { - matches!( - e.op, - ChangeOp::Insert | ChangeOp::Update | ChangeOp::Snapshot - ) - }) - .collect(); + let mut data_events: Vec<&CdcEvent> = Vec::new(); + let mut positions: HashMap>, usize> = HashMap::new(); + + for event in events.iter().filter(|e| { + matches!( + e.op, + ChangeOp::Insert | ChangeOp::Update | ChangeOp::Snapshot + ) + }) { + match row_key(event) { + Some(key) => match positions.get(&key) { + Some(&pos) => data_events[pos] = event, + None => { + positions.insert(key, data_events.len()); + data_events.push(event); + } + }, + None => data_events.push(event), + } + } if data_events.is_empty() { return Ok(None); @@ -966,6 +999,119 @@ mod tests { assert_eq!(pk.value(0), "[\"id\"]"); } + #[test] + fn test_repl_batch_rejects_value_of_wrong_type() { + let schema = ArrowSchema::new(vec![Field::new("age", DataType::Int64, true)]); + + let events = vec![CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([( + "age".into(), + ColumnValue::Text("thirty".into()), + )])), + old: None, + primary_key_columns: vec![], + }]; + + let err = events_to_replication_data_batch(&events, &schema).unwrap_err(); + assert!( + err.to_string().contains("thirty"), + "error should name the offending value: {err}" + ); + } + + #[test] + fn test_repl_batch_accepts_null_for_typed_column() { + let schema = ArrowSchema::new(vec![Field::new("age", DataType::Int64, true)]); + + let events = vec![CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([("age".into(), ColumnValue::Null)])), + old: None, + primary_key_columns: vec![], + }]; + + let batch = events_to_replication_data_batch(&events, &schema) + .unwrap() + .unwrap(); + assert!(batch.column(0).is_null(0)); + } + + #[test] + fn test_repl_batch_keeps_only_last_event_per_pk() { + let schema = ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ]); + + let event = |op, id: i64, name: &str| CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op, + new: Some(BTreeMap::from([ + ("id".into(), ColumnValue::Int(id)), + ("name".into(), ColumnValue::Text(name.into())), + ])), + old: None, + primary_key_columns: vec!["id".into()], + }; + + let events = vec![ + event(ChangeOp::Insert, 1, "first"), + event(ChangeOp::Update, 1, "second"), + event(ChangeOp::Update, 2, "other"), + event(ChangeOp::Update, 1, "third"), + ]; + + let batch = events_to_replication_data_batch(&events, &schema) + .unwrap() + .unwrap(); + + assert_eq!(batch.num_rows(), 2); + let names = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(names.value(0), "third"); + assert_eq!(names.value(1), "other"); + } + + #[test] + fn test_repl_batch_keeps_all_events_without_pk_columns() { + let schema = ArrowSchema::new(vec![Field::new("name", DataType::Utf8, true)]); + + let event = |name: &str| CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([( + "name".into(), + ColumnValue::Text(name.into()), + )])), + old: None, + primary_key_columns: vec![], + }; + + let batch = events_to_replication_data_batch(&[event("a"), event("b")], &schema) + .unwrap() + .unwrap(); + + assert_eq!(batch.num_rows(), 2); + } + // ── Type conversion tests (via replication data batch) ────────── #[test] diff --git a/src/sink/postgres/mod.rs b/src/sink/postgres/mod.rs index d1434ac..6540913 100644 --- a/src/sink/postgres/mod.rs +++ b/src/sink/postgres/mod.rs @@ -16,6 +16,7 @@ use schema::{ }; use crate::schema::value::column_value_to_string; +use crate::sink::postgres::schema::quote_ident; /// Cached information about a target table. struct TableInfo { @@ -78,7 +79,11 @@ impl PostgresSink { let target_table = format!("{}{}", self.config.table_prefix, source_table); // Clone to avoid borrowing self.config while we need &mut self later. let target_schema = self.config.schema.clone(); - let target_fqn = format!("\"{}\".\"{}\"", target_schema, target_table); + let target_fqn = format!( + "{}.{}", + quote_ident(&target_schema), + quote_ident(&target_table) + ); match self.mode { SinkMode::Cdc => { @@ -370,12 +375,12 @@ impl PostgresSink { for col in &info.columns { let val = event.new.as_ref().and_then(|r| r.get(col)); - values.push(column_value_to_string(val)); + values.push(param_value(val)); } for col in &info.columns { let val = event.old.as_ref().and_then(|r| r.get(col)); - values.push(column_value_to_string(val)); + values.push(param_value(val)); } let params: Vec<&(dyn ToSql + Sync)> = @@ -628,7 +633,7 @@ fn build_upsert_sql( pk_columns: &[String], column_types: &HashMap, ) -> String { - let col_list: Vec = columns.iter().map(|c| format!("\"{}\"", c)).collect(); + let col_list: Vec = columns.iter().map(|c| quote_ident(c)).collect(); let placeholders: Vec = columns .iter() .enumerate() @@ -643,10 +648,10 @@ fn build_upsert_sql( .filter(|(_, c)| !pk_columns.contains(c)) .map(|(i, c)| { let typ = column_types.get(c).map(|t| t.as_str()).unwrap_or("text"); - format!("\"{}\" = ${}::{}", c, i + 1, typ) + format!("{} = ${}::{}", quote_ident(c), i + 1, typ) }) .collect(); - let pk_list: Vec = pk_columns.iter().map(|c| format!("\"{}\"", c)).collect(); + let pk_list: Vec = pk_columns.iter().map(|c| quote_ident(c)).collect(); if update_sets.is_empty() { format!( @@ -682,7 +687,7 @@ fn build_delete_sql( .enumerate() .map(|(i, c)| { let typ = column_types.get(c).map(|t| t.as_str()).unwrap_or("text"); - format!("\"{}\" = ${}::{}", c, i + 1, typ) + format!("{} = ${}::{}", quote_ident(c), i + 1, typ) }) .collect(); format!( @@ -692,6 +697,15 @@ fn build_delete_sql( ) } +/// Params go over the wire as TEXT and are cast server-side, so bytes need the +/// `\x` hex prefix — bare hex is read as the escape format and stored literally. +fn param_value(value: Option<&ColumnValue>) -> Option { + match value { + Some(ColumnValue::Bytes(_)) => column_value_to_string(value).map(|hex| format!("\\x{hex}")), + _ => column_value_to_string(value), + } +} + /// UPSERT a row: INSERT ... ON CONFLICT (pk) DO UPDATE SET ... /// /// Uses `prepare_typed` with all `Type::TEXT` so tokio-postgres sends params @@ -704,10 +718,7 @@ async fn upsert_row( ) -> Result<()> { let cols = &info.columns; - let values: Vec> = cols - .iter() - .map(|c| crate::schema::value::column_value_to_string(row.get(c))) - .collect(); + let values: Vec> = cols.iter().map(|c| param_value(row.get(c))).collect(); let sql = build_upsert_sql(&info.target_fqn, cols, &info.pk_columns, &info.column_types); @@ -731,10 +742,7 @@ async fn delete_row( ) -> Result<()> { let pk_cols = &info.pk_columns; - let pk_values: Vec> = pk_cols - .iter() - .map(|c| crate::schema::value::column_value_to_string(row.get(c))) - .collect(); + let pk_values: Vec> = pk_cols.iter().map(|c| param_value(row.get(c))).collect(); let sql = build_delete_sql(&info.target_fqn, pk_cols, &info.column_types); @@ -762,6 +770,23 @@ fn connection_string(config: &PostgresSinkConfig) -> String { mod tests { use super::*; + #[test] + fn test_bytes_param_uses_hex_bytea_literal() { + assert_eq!( + param_value(Some(&ColumnValue::Bytes(vec![0x68, 0x00, 0xff]))), + Some("\\x6800ff".to_string()) + ); + } + + #[test] + fn test_non_bytes_params_are_unchanged() { + assert_eq!( + param_value(Some(&ColumnValue::Text("\\x41".into()))), + Some("\\x41".to_string()) + ); + assert_eq!(param_value(Some(&ColumnValue::Null)), None); + } + #[test] fn test_connection_string_with_password() { let config = PostgresSinkConfig { diff --git a/src/sink/postgres/schema.rs b/src/sink/postgres/schema.rs index 6f2079e..5323df1 100644 --- a/src/sink/postgres/schema.rs +++ b/src/sink/postgres/schema.rs @@ -41,9 +41,15 @@ pub fn canonical_to_pg_ddl(ct: &CanonicalType) -> String { } } +/// Quote an identifier. MongoDB field names may contain `"`, which would +/// otherwise terminate the quoted identifier and let the rest run as SQL. +pub fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + /// Build a fully qualified table name: "schema"."table" fn build_fqn(schema: &str, table: &str) -> String { - format!("\"{}\".\"{}\"", schema, table) + format!("{}.{}", quote_ident(schema), quote_ident(table)) } /// Build the DDL string for creating a CDC-mode table with flattened typed columns. @@ -66,11 +72,11 @@ fn build_cdc_table_ddl(schema: &str, table: &str, source_columns: &[(String, Str ]; for (name, typ) in source_columns { - col_defs.push(format!("\"{}\" {}", name, typ)); + col_defs.push(format!("{} {}", quote_ident(name), typ)); } for (name, typ) in source_columns { - col_defs.push(format!("\"_old_{}\" {}", name, typ)); + col_defs.push(format!("{} {}", quote_ident(&format!("_old_{name}")), typ)); } format!( @@ -112,14 +118,14 @@ fn build_cdc_insert_sql( let mut idx = 8; for col in source_columns { - col_names.push(format!("\"{}\"", col)); + col_names.push(quote_ident(col)); let typ = column_types.get(col).map(|t| t.as_str()).unwrap_or("text"); placeholders.push(format!("${}::{}", idx, typ)); idx += 1; } for col in source_columns { - col_names.push(format!("\"_old_{}\"", col)); + col_names.push(quote_ident(&format!("_old_{col}"))); let typ = column_types.get(col).map(|t| t.as_str()).unwrap_or("text"); placeholders.push(format!("${}::{}", idx, typ)); idx += 1; @@ -145,9 +151,9 @@ fn build_replication_table_ddl( let fqn = build_fqn(schema, table); let col_defs: Vec = columns .iter() - .map(|(name, typ)| format!("\"{}\" {}", name, typ)) + .map(|(name, typ)| format!("{} {}", quote_ident(name), typ)) .collect(); - let pk_list: Vec = pk_columns.iter().map(|c| format!("\"{}\"", c)).collect(); + let pk_list: Vec = pk_columns.iter().map(|c| quote_ident(c)).collect(); format!( "CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))", fqn, @@ -163,7 +169,7 @@ fn build_add_columns_ddl(schema: &str, table: &str, new_columns: &[(String, Stri let fqn = build_fqn(schema, table); let alter_clauses: Vec = new_columns .iter() - .map(|(name, typ)| format!("ADD COLUMN IF NOT EXISTS \"{}\" {}", name, typ)) + .map(|(name, typ)| format!("ADD COLUMN IF NOT EXISTS {} {}", quote_ident(name), typ)) .collect(); format!("ALTER TABLE {} {}", fqn, alter_clauses.join(", ")) } @@ -175,7 +181,7 @@ fn build_drop_columns_ddl(schema: &str, table: &str, columns: &[String]) -> Stri let fqn = build_fqn(schema, table); let alter_clauses: Vec = columns .iter() - .map(|name| format!("DROP COLUMN IF EXISTS \"{}\"", name)) + .map(|name| format!("DROP COLUMN IF EXISTS {}", quote_ident(name))) .collect(); format!("ALTER TABLE {} {}", fqn, alter_clauses.join(", ")) } @@ -330,6 +336,27 @@ pub async fn add_columns( mod tests { use super::*; + #[test] + fn test_quote_ident_escapes_embedded_quotes() { + assert_eq!(quote_ident("bad\"name"), "\"bad\"\"name\""); + } + + #[test] + fn test_ddl_with_hostile_field_name_stays_one_identifier() { + let ddl = build_cdc_table_ddl( + "public", + "weird", + &[( + "x\" text); DROP TABLE t; --".to_string(), + "text".to_string(), + )], + ); + assert!( + ddl.contains("\"x\"\" text); DROP TABLE t; --\" text"), + "field name must stay inside one quoted identifier: {ddl}" + ); + } + #[test] fn test_build_fqn() { assert_eq!(build_fqn("public", "users"), "\"public\".\"users\""); diff --git a/src/source/mongodb/change_stream.rs b/src/source/mongodb/change_stream.rs index ffbec1b..95ea282 100644 --- a/src/source/mongodb/change_stream.rs +++ b/src/source/mongodb/change_stream.rs @@ -67,7 +67,22 @@ pub async fn open_change_stream( builder .await - .map_err(|e| CdcError::Mongodb(format!("failed to open change stream: {e}"))) + .map_err(|e| stream_error("failed to open change stream", e.to_string())) +} + +/// An update to a large document carries both a post- and a pre-image, so the +/// event can exceed the 16 MB BSON limit even though each document is legal. +/// It then replays on every restart, which the driver's own text does not hint at. +fn stream_error(context: &str, detail: String) -> CdcError { + let mut msg = format!("{context}: {detail}"); + if detail.contains("BSONObjectTooLarge") { + msg.push_str( + " — the change event exceeded MongoDB's 16 MB limit, most likely an update to a \ + large document with changeStreamPreAndPostImages enabled. The event replays on \ + every restart: disable pre-images for the collection and re-snapshot to get past it.", + ); + } + CdcError::Mongodb(msg) } /// True if the start position has aged out of the oplog — it can never become @@ -161,7 +176,7 @@ pub async fn pump_change_stream( } } Err(e) => { - return Err(CdcError::Mongodb(format!("change stream error: {e}"))); + return Err(stream_error("change stream error", e.to_string())); } } } @@ -200,6 +215,24 @@ mod tests { } } + #[test] + fn test_oversized_event_error_explains_pre_images() { + let err = stream_error( + "change stream error", + "Error code 10334 (BSONObjectTooLarge): Size 28312133 exceeds maximum 16793600".into(), + ); + assert!(err.to_string().contains("pre-images"), "{err}"); + } + + #[test] + fn test_unrelated_errors_get_no_extra_hint() { + let err = stream_error("change stream error", "connection refused".into()); + assert_eq!( + err.to_string(), + "mongodb error: change stream error: connection refused" + ); + } + #[test] fn test_other_errors_are_not_treated_as_lost_resume_points() { let cases = [ diff --git a/tests/postgres_sink_integration.rs b/tests/postgres_sink_integration.rs index 7d94820..97b8eae 100644 --- a/tests/postgres_sink_integration.rs +++ b/tests/postgres_sink_integration.rs @@ -1206,3 +1206,57 @@ async fn test_replication_mode_mongodb_sparse_field_does_not_drop_column() { "a field missing from one MongoDB batch must not drop the target column" ); } + +#[tokio::test] +#[ignore] +async fn test_cdc_mode_binary_column_round_trips() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"files\" ( + \"_cdc_op\" text, + \"_cdc_lsn\" bigint, + \"_cdc_timestamp_us\" bigint, + \"_cdc_snapshot\" boolean, + \"_cdc_schema\" text, + \"_cdc_table\" text, + \"_cdc_primary_key_columns\" text, + \"id\" text, + \"blob\" bytea, + \"_old_id\" text, + \"_old_blob\" bytea + )", + &[], + ) + .await + .unwrap(); + + let config = make_sink_config(&host, port); + let mut sink = PostgresSink::new(config, SinkMode::Cdc, None) + .await + .unwrap(); + + let payload = vec![0x00u8, 0x68, 0x69, 0xff]; + let event = make_insert_event( + "demo", + "files", + vec![ + ("id", ColumnValue::Text("f1".into())), + ("blob", ColumnValue::Bytes(payload.clone())), + ], + ); + + sink.write_batch(&[event]).await.unwrap(); + sink.flush().await.unwrap(); + + let stored: Vec = client + .query_one("SELECT \"blob\" FROM \"public\".\"files\"", &[]) + .await + .unwrap() + .get(0); + + assert_eq!(stored, payload); +}