diff --git a/Cargo.lock b/Cargo.lock index 5ec5631..aa26b54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,6 +681,29 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash 0.8.12", + "base64 0.22.1", + "bitvec", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap 2.13.0", + "js-sys", + "once_cell", + "rand 0.9.2", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "btoi" version = "0.4.3" @@ -763,6 +786,7 @@ dependencies = [ "futures-util", "iceberg", "iceberg-catalog-rest", + "mongodb", "mysql_async", "nix", "ordered-float 5.1.0", @@ -919,6 +943,15 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -962,6 +995,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -1115,6 +1154,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + [[package]] name = "deranged" version = "0.5.8" @@ -1125,6 +1170,28 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -1156,6 +1223,29 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -1244,6 +1334,18 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1651,6 +1753,52 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.9.2", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "moka", + "once_cell", + "parking_lot", + "rand 0.9.2", + "resolv-conf", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1894,7 +2042,7 @@ dependencies = [ "serde_with", "strum", "tokio", - "typed-builder", + "typed-builder 0.20.1", "url", "uuid", "zstd", @@ -1917,7 +2065,7 @@ dependencies = [ "serde_json", "tokio", "tracing", - "typed-builder", + "typed-builder 0.20.1", "uuid", ] @@ -2064,6 +2212,18 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2 0.5.10", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2368,6 +2528,54 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.117", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2446,6 +2654,82 @@ dependencies = [ "uuid", ] +[[package]] +name = "mongocrypt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.5+1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" + +[[package]] +name = "mongodb" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803dd859e8afa084c255a8effd8000ff86f7c8076a50cd6d8c99e8f3496f75c2" +dependencies = [ + "base64 0.22.1", + "bitflags", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-proto", + "hickory-resolver", + "hmac", + "macro_magic", + "md-5", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2", + "percent-encoding", + "rand 0.9.2", + "rustc_version_runtime", + "rustls", + "rustversion", + "serde", + "serde_bytes", + "serde_with", + "sha1", + "sha2", + "socket2 0.6.2", + "stringprep", + "strsim", + "take_mut", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "typed-builder 0.22.0", + "uuid", + "webpki-roots", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a973ef3dd3dbc6f6e65bbdecfd9ec5e781b9e7493b0f369a7c62e35d8e5ae2c8" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "murmur3" version = "0.5.2" @@ -2674,6 +2958,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2857,6 +3145,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", +] + [[package]] name = "pem" version = "3.0.6" @@ -3497,6 +3794,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "ring" version = "0.17.14" @@ -3616,6 +3919,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3859,6 +4172,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap 2.13.0", "itoa", "memchr", "serde", @@ -4167,6 +4481,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + [[package]] name = "tap" version = "1.0.1" @@ -4443,6 +4763,7 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -4648,7 +4969,16 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.20.1", +] + +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro 0.22.0", ] [[package]] @@ -4662,6 +4992,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "typenum" version = "1.19.0" @@ -4695,6 +5036,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -5013,6 +5360,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -5112,6 +5465,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5154,6 +5516,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -5193,6 +5570,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5211,6 +5594,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5229,6 +5618,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5259,6 +5654,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -5277,6 +5678,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -5295,6 +5702,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -5313,6 +5726,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -5334,6 +5753,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/Cargo.toml b/Cargo.toml index 0888a08..dc7e4fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ description = "A Change Data Capture (CDC) pipeline in Rust" repository = "https://github.com/manfredcml/cdcflow" homepage = "https://github.com/manfredcml/cdcflow" readme = "README.md" -keywords = ["cdc", "change-data-capture", "replication", "postgres", "mysql"] +keywords = ["cdc", "change-data-capture", "replication", "postgres", "mongodb"] categories = ["database", "command-line-utilities"] exclude = ["benchmark/"] @@ -33,6 +33,7 @@ arrow-schema = "57" parquet = { version = "57", default-features = false, features = ["arrow"] } async-trait = "0.1" rdkafka = { version = "0.39.0", features = ["cmake-build", "tokio"] } +mongodb = "3" ordered-float = { version = "5.1.0", features = ["serde"] } chrono = "0.4" axum = { version = "0.8", features = ["json"] } @@ -50,4 +51,4 @@ approx_constant = "allow" tower = "0.5" tempfile = "3" testcontainers = "0.27.1" -testcontainers-modules = { version = "0.15.0", features = ["postgres", "mysql", "kafka"] } +testcontainers-modules = { version = "0.15.0", features = ["postgres", "mysql", "kafka", "mongo"] } diff --git a/README.md b/README.md index ebeb925..8e5b309 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ destinations. ## Features -- **Database sources**: PostgreSQL and MySQL +- **Database sources**: PostgreSQL, MySQL, and MongoDB - **Sink destinations**: Stdout, Kafka, PostgreSQL, Apache Iceberg - **Offset stores**: SQLite, Memoryfor resumable streaming after restarts - **Sync modes**: CDC (append-only changelog) and Replication (live replica of source tables) @@ -20,7 +20,7 @@ destinations. ``` ┌────────────┐ ┌────────────┐ ┌──────────────┐ │ Source │────▶│ Pipeline │────▶│ Sink │ -│ PG / MySQL │ │ (batch + │ │ Kafka / PG / │ +│PG/MySQL/Mon│ │ (batch + │ │ Kafka / PG / │ │ │ │ flush) │ │ Iceberg / Out│ └────────────┘ └─────┬──────┘ └──────────────┘ │ @@ -37,12 +37,32 @@ checkpoints progress in the offset store. On restart, the pipeline resumes from - **Platform**: Linux and macOS only - **CMake**: Required for building the bundled librdkafka (Kafka dependency) +### Source requirements + +| Source | Requirements | +| --- | --- | +| PostgreSQL | Logical replication (`wal_level=logical`), a replication slot and publication | +| MySQL | Row-based binlog (`binlog_format=ROW`, `binlog_row_image=FULL`) | +| MongoDB | Replica set (or sharded cluster) — change streams do not work on standalone `mongod` | + +**MongoDB pre-images (optional).** Enabling `changeStreamPreAndPostImages` on a +collection makes the full "before" row available on updates and deletes: + +```js +db.runCommand({ collMod: "users", changeStreamPreAndPostImages: { enabled: true } }); +``` + +Without it, update and delete events fall back to the change event's +`documentKey`, so the `_id` primary key is always captured — replication mode +still applies updates and deletes correctly, but CDC-mode `old` values contain +only `_id`. + ## Quick Start - Standalone Mode The standalone mode runs a single pipeline without the admin server. This is ideal for local testing and development. ```bash -# Start local infrastructure (PostgreSQL, MySQL, Kafka, MinIO, Iceberg, Trino) +# Start local infrastructure (PostgreSQL, MySQL, MongoDB, Kafka, MinIO, Iceberg, Trino) # You can optionally start just the components you need for testing a specific source/sink combo cd example && docker compose up -d && cd .. diff --git a/example/configs/mongo-to-iceberg-replication.json b/example/configs/mongo-to-iceberg-replication.json new file mode 100644 index 0000000..e67ccb5 --- /dev/null +++ b/example/configs/mongo-to-iceberg-replication.json @@ -0,0 +1,31 @@ +{ + "mode": "replication", + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "iceberg", + "catalog": { + "type": "rest", + "uri": "http://localhost:8181", + "warehouse": "s3://warehouse/iceberg", + "properties": { + "s3.endpoint": "http://localhost:9000", + "s3.access-key-id": "minioadmin", + "s3.secret-access-key": "minioadmin", + "s3.path-style-access": "true", + "s3.region": "us-east-1" + } + }, + "namespace": ["demo"], + "table_prefix": "" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-iceberg-replication" + } +} diff --git a/example/configs/mongo-to-iceberg.json b/example/configs/mongo-to-iceberg.json new file mode 100644 index 0000000..29c8d63 --- /dev/null +++ b/example/configs/mongo-to-iceberg.json @@ -0,0 +1,30 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "iceberg", + "catalog": { + "type": "rest", + "uri": "http://localhost:8181", + "warehouse": "s3://warehouse/iceberg", + "properties": { + "s3.endpoint": "http://localhost:9000", + "s3.access-key-id": "minioadmin", + "s3.secret-access-key": "minioadmin", + "s3.path-style-access": "true", + "s3.region": "us-east-1" + } + }, + "namespace": ["demo"], + "table_prefix": "" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-iceberg" + } +} diff --git a/example/configs/mongo-to-kafka.json b/example/configs/mongo-to-kafka.json new file mode 100644 index 0000000..c9211f4 --- /dev/null +++ b/example/configs/mongo-to-kafka.json @@ -0,0 +1,18 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "kafka", + "brokers": "localhost:9092", + "topic_prefix": "cdc" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-kafka" + } +} diff --git a/example/configs/mongo-to-pg-replication.json b/example/configs/mongo-to-pg-replication.json new file mode 100644 index 0000000..08c15c3 --- /dev/null +++ b/example/configs/mongo-to-pg-replication.json @@ -0,0 +1,23 @@ +{ + "mode": "replication", + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "postgres", + "host": "localhost", + "port": 5433, + "user": "cdc_user", + "password": "cdc_password", + "database": "demo_dest", + "schema": "public" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-pg-replication" + } +} diff --git a/example/configs/mongo-to-pg.json b/example/configs/mongo-to-pg.json new file mode 100644 index 0000000..f19c5cc --- /dev/null +++ b/example/configs/mongo-to-pg.json @@ -0,0 +1,22 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "postgres", + "host": "localhost", + "port": 5433, + "user": "cdc_user", + "password": "cdc_password", + "database": "demo_dest", + "schema": "public" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-pg" + } +} diff --git a/example/configs/mongo-to-stdout.json b/example/configs/mongo-to-stdout.json new file mode 100644 index 0000000..433a4eb --- /dev/null +++ b/example/configs/mongo-to-stdout.json @@ -0,0 +1,16 @@ +{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true", + "database": "demo", + "collections": ["users", "orders"] + }, + "sink": { + "type": "stdout" + }, + "offset": { + "type": "sqlite", + "path": "/tmp/cdc-offsets.db", + "key": "mongo-stdout" + } +} diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 19b223a..a0e147e 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -50,6 +50,28 @@ services: timeout: 5s retries: 10 + mongodb: + image: mongo:8 + container_name: cdc-mongodb + ports: + - "27017:27017" + command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] + healthcheck: + test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 5s + retries: 10 + + mongodb-init: + image: mongo:8 + container_name: cdc-mongodb-init + depends_on: + mongodb: + condition: service_healthy + volumes: + - ./init/mongodb/init.js:/init.js + entrypoint: ["mongosh", "mongodb://mongodb:27017/?directConnection=true", "--file", "/init.js"] + # ────────────────────────────────────────────── # Infrastructure # ────────────────────────────────────────────── diff --git a/example/init/mongodb/init.js b/example/init/mongodb/init.js new file mode 100644 index 0000000..f00314a --- /dev/null +++ b/example/init/mongodb/init.js @@ -0,0 +1,56 @@ +// Initialize replica set (required for change streams). +// Re-running the script against an already-initialized set is not an error. +try { + rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost:27017" }] }); +} catch (e) { + print("replSetInitiate: " + e); +} + +// Wait for primary election +let attempts = 0; +while (!db.hello().isWritablePrimary && attempts < 30) { + sleep(1000); + attempts++; +} + +if (!db.hello().isWritablePrimary) { + print("ERROR: Failed to elect primary after 30 seconds"); + quit(1); +} + +print("Replica set initialized, primary elected"); + +// Create demo database and collections. +// NOTE: `const db = db.getSiblingDB(...)` is a self-referencing declaration and +// throws a ReferenceError — the handle must use a different name. +const demoDb = db.getSiblingDB("demo"); + +// Enable pre/post images for change streams (MongoDB 6.0+). +// Optional: without them, update/delete events fall back to `documentKey`, so +// the `_id` primary key is still captured but the full "before" row is not. +demoDb.createCollection("users", { + changeStreamPreAndPostImages: { enabled: true }, +}); +demoDb.createCollection("orders", { + changeStreamPreAndPostImages: { enabled: true }, +}); + +// Insert sample data +demoDb.users.insertMany([ + { _id: ObjectId(), name: "Alice", email: "alice@example.com", age: 30, active: true }, + { _id: ObjectId(), name: "Bob", email: "bob@example.com", age: 25, active: true }, + { _id: ObjectId(), name: "Charlie", email: "charlie@example.com", age: 35, active: false }, +]); + +demoDb.orders.insertMany([ + { _id: ObjectId(), user: "Alice", product: "Widget", quantity: 2, price: 19.99 }, + { _id: ObjectId(), user: "Bob", product: "Gadget", quantity: 1, price: 49.99 }, +]); + +print( + "Demo data inserted: " + + demoDb.users.countDocuments() + + " users, " + + demoDb.orders.countDocuments() + + " orders", +); diff --git a/src/config/mod.rs b/src/config/mod.rs index f532d47..08a9c13 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,5 +1,6 @@ mod iceberg_sink; mod kafka_sink; +mod mongodb_source; mod mysql_source; mod offset; mod postgres_sink; @@ -9,6 +10,7 @@ mod source; pub use self::iceberg_sink::{IcebergCatalogConfig, IcebergSinkConfig, RestCatalogConfig}; pub use self::kafka_sink::KafkaSinkConfig; +pub use self::mongodb_source::MongodbSourceConfig; pub use self::mysql_source::MySqlSourceConfig; pub use self::offset::OffsetConfig; pub use self::postgres_sink::PostgresSinkConfig; @@ -37,6 +39,8 @@ pub enum SourceConnectionConfig { Postgres { url: String }, #[serde(rename = "mysql")] Mysql { url: String }, + #[serde(rename = "mongodb")] + Mongodb { url: String }, } /// Top-level configuration for the CDC agent. @@ -53,6 +57,28 @@ pub struct Config { mod tests { use super::*; + /// Every shipped example config must deserialize, so a config that drifts + /// from the schema (wrong field type, renamed key) fails the build rather + /// than only failing when a user runs it. + #[test] + fn test_example_configs_deserialize() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("example/configs"); + let mut checked = 0; + + for entry in std::fs::read_dir(&dir).expect("example/configs must exist") { + let path = entry.expect("readable dir entry").path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + let contents = std::fs::read_to_string(&path).expect("readable config"); + serde_json::from_str::(&contents) + .unwrap_or_else(|e| panic!("{} failed to parse: {e}", path.display())); + checked += 1; + } + + assert!(checked > 0, "no example configs found in {}", dir.display()); + } + #[test] fn test_sink_mode_default_is_cdc() { let json = r#"{ diff --git a/src/config/mongodb_source.rs b/src/config/mongodb_source.rs new file mode 100644 index 0000000..7e50ab3 --- /dev/null +++ b/src/config/mongodb_source.rs @@ -0,0 +1,66 @@ +use serde::{Deserialize, Serialize}; + +/// MongoDB-specific source configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MongodbSourceConfig { + /// MongoDB connection string (e.g. `mongodb://host:port/?replicaSet=rs0`). + pub connection_string: String, + /// Database to capture changes from. + pub database: String, + /// Collections to capture. Empty means all collections. + #[serde(default)] + pub collections: Vec, +} + +#[cfg(test)] +mod tests { + use crate::config::{Config, SourceConfig}; + + #[test] + fn test_mongodb_config_deserialization() { + let json = r#"{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "mydb", + "collections": ["users", "orders"] + }, + "sink": { "type": "stdout" }, + "offset": { "type": "memory" } + }"#; + + let config: Config = serde_json::from_str(json).unwrap(); + match &config.source { + SourceConfig::Mongodb(mg) => { + assert_eq!( + mg.connection_string, + "mongodb://localhost:27017/?replicaSet=rs0" + ); + assert_eq!(mg.database, "mydb"); + assert_eq!(mg.collections, vec!["users", "orders"]); + } + _ => panic!("expected Mongodb config"), + } + } + + #[test] + fn test_mongodb_config_empty_collections_default() { + let json = r#"{ + "source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "database": "mydb" + }, + "sink": { "type": "stdout" }, + "offset": { "type": "sqlite", "path": "/tmp/offset.db", "key": "mongo-offset" } + }"#; + + let config: Config = serde_json::from_str(json).unwrap(); + match &config.source { + SourceConfig::Mongodb(mg) => { + assert!(mg.collections.is_empty()); + } + _ => panic!("expected Mongodb config"), + } + } +} diff --git a/src/config/source.rs b/src/config/source.rs index 9f6063c..d66ddc4 100644 --- a/src/config/source.rs +++ b/src/config/source.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; +use super::mongodb_source::MongodbSourceConfig; use super::mysql_source::MySqlSourceConfig; use super::postgres_source::PostgresSourceConfig; @@ -11,6 +12,8 @@ pub enum SourceConfig { Postgres(PostgresSourceConfig), #[serde(rename = "mysql")] Mysql(MySqlSourceConfig), + #[serde(rename = "mongodb")] + Mongodb(MongodbSourceConfig), } impl SourceConfig { @@ -47,6 +50,9 @@ impl SourceConfig { my.database, ), }, + SourceConfig::Mongodb(mg) => super::SourceConnectionConfig::Mongodb { + url: mg.connection_string.clone(), + }, } } } @@ -120,6 +126,21 @@ mod tests { } } + #[test] + fn test_mongodb_to_connection_config() { + let config = SourceConfig::Mongodb(super::MongodbSourceConfig { + connection_string: "mongodb://localhost:27017/?replicaSet=rs0".into(), + database: "mydb".into(), + collections: vec![], + }); + match config.to_connection_config() { + SourceConnectionConfig::Mongodb { url } => { + assert_eq!(url, "mongodb://localhost:27017/?replicaSet=rs0"); + } + _ => panic!("expected Mongodb variant"), + } + } + #[test] fn test_mysql_to_connection_config_without_password() { let config = SourceConfig::Mysql(MySqlSourceConfig { diff --git a/src/error.rs b/src/error.rs index 4f6ee5b..b823cb3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,7 +2,7 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum CdcError { - #[error("postgres error: {0}")] + #[error("postgres error: {}", format_postgres_error(.0))] Postgres(#[from] tokio_postgres::Error), #[error("protocol error: {0}")] @@ -26,6 +26,9 @@ pub enum CdcError { #[error("mysql error: {0}")] Mysql(String), + #[error("mongodb error: {0}")] + Mongodb(String), + #[error("iceberg error: {0}")] Iceberg(String), @@ -45,4 +48,22 @@ pub enum CdcError { Schema(String), } +/// `tokio_postgres::Error` renders server failures as the bare string +/// "db error" — the useful text only lives in the wrapped `DbError`. +fn format_postgres_error(err: &tokio_postgres::Error) -> String { + match err.as_db_error() { + Some(db) => { + let mut msg = format!("{}: {}", db.severity(), db.message()); + if let Some(detail) = db.detail() { + msg.push_str(&format!(" (detail: {detail})")); + } + if let Some(column) = db.column() { + msg.push_str(&format!(" (column: {column})")); + } + msg + } + None => err.to_string(), + } +} + pub type Result = std::result::Result; diff --git a/src/lib.rs b/src/lib.rs index 5458342..6272e53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ //! # cdcflow //! //! A Change Data Capture (CDC) pipeline that captures row-level changes -//! from PostgreSQL and MySQL, streaming them to Kafka, PostgreSQL, +//! from PostgreSQL, MySQL, and MongoDB, streaming them to Kafka, PostgreSQL, //! Apache Iceberg, or stdout. //! //! The pipeline reads events from a [`source`], batches them by transaction diff --git a/src/main.rs b/src/main.rs index e89d869..b684dd9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ use cdcflow::sink::kafka::KafkaSink; use cdcflow::sink::postgres::PostgresSink; use cdcflow::sink::stdout::StdoutSink; use cdcflow::sink::Sink; +use cdcflow::source::mongodb::MongodbSource; use cdcflow::source::mysql::MySqlSource; use cdcflow::source::postgres::PostgresSource; use cdcflow::worker_http::registration::AdminClient; @@ -23,7 +24,7 @@ use cdcflow::worker_http::WorkerHttpServer; #[derive(Parser, Debug)] #[command( name = "cdcflow", - about = "Change Data Capture agent for PostgreSQL and MySQL" + about = "Change Data Capture agent for PostgreSQL, MySQL, and MongoDB" )] struct Cli { #[command(subcommand)] @@ -646,6 +647,26 @@ async fn run_with_sink( pipeline.run(shutdown).await } }, + SourceConfig::Mongodb(mg_config) => match offset_config { + OffsetConfig::Sqlite { path, key } => { + let os = SqliteOffsetStore::new(&path, &key)?; + let mut pipeline = + Pipeline::new(MongodbSource::new(mg_config, os.clone()), sink, os); + if let Some(m) = metrics { + pipeline = pipeline.with_metrics(m); + } + pipeline.run(shutdown).await + } + OffsetConfig::Memory => { + let os = MemoryOffsetStore::new(); + let mut pipeline = + Pipeline::new(MongodbSource::new(mg_config, os.clone()), sink, os); + if let Some(m) = metrics { + pipeline = pipeline.with_metrics(m); + } + pipeline.run(shutdown).await + } + }, } } diff --git a/src/schema/discovery.rs b/src/schema/discovery.rs index 35dbc11..9c3208c 100644 --- a/src/schema/discovery.rs +++ b/src/schema/discovery.rs @@ -18,6 +18,7 @@ pub async fn fetch_columns( fetch_columns_postgres(url, schema, table).await } SourceConnectionConfig::Mysql { url } => fetch_columns_mysql(url, schema, table).await, + SourceConnectionConfig::Mongodb { url } => fetch_columns_mongodb(url, schema, table).await, } } @@ -32,6 +33,7 @@ pub async fn fetch_primary_keys( fetch_primary_keys_postgres(url, schema, table).await } SourceConnectionConfig::Mysql { url } => fetch_primary_keys_mysql(url, schema, table).await, + SourceConnectionConfig::Mongodb { .. } => Ok(vec!["_id".to_string()]), } } @@ -52,6 +54,9 @@ pub async fn fetch_column_subset( SourceConnectionConfig::Mysql { url } => { fetch_column_subset_mysql(url, schema, table, column_names).await } + SourceConnectionConfig::Mongodb { url } => { + fetch_column_subset_mongodb(url, schema, table, column_names).await + } } } @@ -340,3 +345,124 @@ async fn fetch_column_subset_mysql( Ok(result) } + +const SCHEMA_SAMPLE_SIZE: i64 = 100; + +/// Cached clients keyed by connection string — discovery runs per write batch, +/// and building a `mongodb::Client` re-runs topology discovery each time. +static MONGO_CLIENTS: std::sync::OnceLock< + tokio::sync::Mutex>, +> = std::sync::OnceLock::new(); + +async fn mongodb_client(url: &str) -> Result { + let mut clients = MONGO_CLIENTS + .get_or_init(|| tokio::sync::Mutex::new(std::collections::HashMap::new())) + .lock() + .await; + + if let Some(client) = clients.get(url) { + return Ok(client.clone()); + } + + let client = mongodb::Client::with_uri_str(url) + .await + .map_err(|e| CdcError::Schema(format!("mongodb connect: {e}")))?; + clients.insert(url.to_string(), client.clone()); + Ok(client) +} + +/// Sample documents from both ends of a collection and merge their field types. +/// +/// Callers treat the result as the current source schema, so both ends matter: +/// the newest documents carry fields added later, the oldest carry fields only +/// they still have. Sorted by `_id` to keep the sample stable across runs. +async fn sample_field_types( + coll: &mongodb::Collection, +) -> Result> { + use futures_util::StreamExt; + use mongodb::bson::doc; + + let mut field_types: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for order in [1, -1] { + let mut cursor = coll + .find(doc! {}) + .sort(doc! { "_id": order }) + .limit(SCHEMA_SAMPLE_SIZE / 2) + .await + .map_err(|e| CdcError::Schema(format!("mongodb find: {e}")))?; + + while let Some(result) = cursor.next().await { + let doc = result.map_err(|e| CdcError::Schema(format!("mongodb cursor: {e}")))?; + crate::source::mongodb::bson_mapping::merge_document_field_types( + &mut field_types, + &doc, + ); + } + } + + Ok(field_types) +} + +/// Fetch columns by sampling documents from a MongoDB collection. +/// +/// Since MongoDB is schemaless, we sample up to `SCHEMA_SAMPLE_SIZE` documents +/// and build a union of all top-level field names. Types observed for the same +/// field across documents are merged (nulls ignored, numerics widened, +/// otherwise falling back to text). +async fn fetch_columns_mongodb( + url: &str, + database: &str, + collection: &str, +) -> Result> { + let client = mongodb_client(url).await?; + let db = client.database(database); + let coll = db.collection::(collection); + + let field_types = sample_field_types(&coll).await?; + + let dialect = SourceDialect::Mongodb; + let columns: Vec = field_types + .into_iter() + .enumerate() + .map(|(i, (name, type_str))| { + let canonical_type = parse_source_type(dialect, &type_str); + ColumnInfo { + name, + source_type: type_str, + canonical_type, + is_nullable: true, + ordinal_position: i as i32, + } + }) + .collect(); + + Ok(columns) +} + +async fn fetch_column_subset_mongodb( + url: &str, + database: &str, + collection: &str, + column_names: &[String], +) -> Result> { + let all_columns = fetch_columns_mongodb(url, database, collection).await?; + let type_map: std::collections::HashMap = all_columns + .into_iter() + .map(|col| (col.name.clone(), col)) + .collect(); + + Ok(column_names + .iter() + .map(|name| { + type_map.get(name).cloned().unwrap_or_else(|| ColumnInfo { + name: name.clone(), + source_type: "string".into(), + canonical_type: super::CanonicalType::Text, + is_nullable: true, + ordinal_position: 0, + }) + }) + .collect()) +} diff --git a/src/schema/evolution.rs b/src/schema/evolution.rs index e989ea0..8d905a7 100644 --- a/src/schema/evolution.rs +++ b/src/schema/evolution.rs @@ -1,5 +1,6 @@ use std::collections::HashSet; +use crate::config::SourceConnectionConfig; use crate::event::{CdcEvent, ChangeOp}; /// Detect column names present in events but missing from the known column list. @@ -30,6 +31,14 @@ pub fn detect_new_columns(known_columns: &[String], events: &[&CdcEvent]) -> Vec new_cols } +/// Whether a column's absence from the source can be treated as a drop. +/// +/// False for schemaless sources: MongoDB discovery only samples documents, so +/// absence there would drop live columns and their replicated data. +pub fn source_supports_column_drops(source_conn: &SourceConnectionConfig) -> bool { + !matches!(source_conn, SourceConnectionConfig::Mongodb { .. }) +} + /// Detect target columns that no longer exist in the source. /// /// Returns column names present in `target_columns` but absent from `source_columns`. @@ -50,6 +59,29 @@ mod tests { use crate::event::{ColumnValue, Lsn, TableId}; use std::collections::BTreeMap; + #[test] + fn test_schemaless_sources_never_confirm_column_drops() { + assert!(!source_supports_column_drops( + &SourceConnectionConfig::Mongodb { + url: "mongodb://localhost:27017".into() + } + )); + } + + #[test] + fn test_relational_sources_confirm_column_drops() { + assert!(source_supports_column_drops( + &SourceConnectionConfig::Postgres { + url: "postgres://localhost/db".into() + } + )); + assert!(source_supports_column_drops( + &SourceConnectionConfig::Mysql { + url: "mysql://localhost/db".into() + } + )); + } + fn make_table_id() -> TableId { TableId { schema: "public".into(), diff --git a/src/schema/mod.rs b/src/schema/mod.rs index def633d..f1c4685 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -11,6 +11,7 @@ use crate::event::ColumnValue; pub enum SourceDialect { Postgres, Mysql, + Mongodb, } impl From<&SourceConnectionConfig> for SourceDialect { @@ -18,6 +19,7 @@ impl From<&SourceConnectionConfig> for SourceDialect { match conn { SourceConnectionConfig::Postgres { .. } => SourceDialect::Postgres, SourceConnectionConfig::Mysql { .. } => SourceDialect::Mysql, + SourceConnectionConfig::Mongodb { .. } => SourceDialect::Mongodb, } } } diff --git a/src/schema/type_mapping.rs b/src/schema/type_mapping.rs index 92affaa..2cffd65 100644 --- a/src/schema/type_mapping.rs +++ b/src/schema/type_mapping.rs @@ -9,6 +9,7 @@ pub fn parse_source_type(dialect: SourceDialect, raw_type: &str) -> CanonicalTyp match dialect { SourceDialect::Postgres => parse_postgres_type(&dt), SourceDialect::Mysql => parse_mysql_type(&dt), + SourceDialect::Mongodb => parse_mongodb_type(&dt), } } @@ -143,6 +144,23 @@ fn parse_mysql_type(dt: &str) -> CanonicalType { } } +fn parse_mongodb_type(dt: &str) -> CanonicalType { + match dt { + "objectid" | "string" => CanonicalType::Text, + // Both arrive as ColumnValue::Int(i64); a 32-bit column would truncate. + "int" | "long" => CanonicalType::BigInt, + "double" => CanonicalType::Double, + "bool" => CanonicalType::Boolean, + "date" => CanonicalType::TimestampTz, + "bindata" => CanonicalType::Binary, + "object" | "array" => CanonicalType::Json, + "decimal" => CanonicalType::Text, // Decimal128 has 34 digits precision, store as text + // A field that was null in every sampled document carries no type info. + "null" => CanonicalType::Text, + _ => CanonicalType::Text, + } +} + #[cfg(test)] mod tests { use super::*; @@ -503,6 +521,47 @@ mod tests { ); } + #[test] + fn test_mongodb_type_mapping() { + let cases = vec![ + ("objectid", CanonicalType::Text), + ("string", CanonicalType::Text), + ("int", CanonicalType::BigInt), + ("long", CanonicalType::BigInt), + ("double", CanonicalType::Double), + ("bool", CanonicalType::Boolean), + ("date", CanonicalType::TimestampTz), + ("bindata", CanonicalType::Binary), + ("object", CanonicalType::Json), + ("array", CanonicalType::Json), + ("decimal", CanonicalType::Text), + ("unknown_bson_type", CanonicalType::Text), + ]; + for (input, expected) in cases { + assert_eq!( + parse_source_type(SourceDialect::Mongodb, input), + expected, + "failed for MongoDB type '{input}'" + ); + } + } + + #[test] + fn test_mongodb_case_insensitive() { + assert_eq!( + parse_source_type(SourceDialect::Mongodb, "ObjectId"), + CanonicalType::Text + ); + assert_eq!( + parse_source_type(SourceDialect::Mongodb, "STRING"), + CanonicalType::Text + ); + assert_eq!( + parse_source_type(SourceDialect::Mongodb, "Int"), + CanonicalType::BigInt + ); + } + #[test] fn test_mysql_case_insensitive() { assert_eq!( diff --git a/src/sink/iceberg/mod.rs b/src/sink/iceberg/mod.rs index 6c57e1c..02f5cbd 100644 --- a/src/sink/iceberg/mod.rs +++ b/src/sink/iceberg/mod.rs @@ -362,6 +362,16 @@ impl IcebergSink { schema_evolution::detect_dropped_columns_from_events(arrow_schema, events, false); if has_full_row_events && !missing_from_events.is_empty() { + if !crate::schema::evolution::source_supports_column_drops(&self.source_connection) { + tracing::debug!( + schema = source_schema, + table = source_table, + ?missing_from_events, + "schemaless source: keeping columns missing from events", + ); + return Ok(()); + } + tracing::debug!( schema = source_schema, table = source_table, diff --git a/src/sink/iceberg/record_batch.rs b/src/sink/iceberg/record_batch.rs index dae1d2e..3877c19 100644 --- a/src/sink/iceberg/record_batch.rs +++ b/src/sink/iceberg/record_batch.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::Arc; use arrow_array::{ @@ -74,6 +75,35 @@ pub fn events_to_record_batch( .map_err(|e| CdcError::Iceberg(format!("record batch: {e}"))) } +/// A source value that is present but cannot be represented in the target +/// column — a schemaless source changed the field's type after the table was +/// created. Writing NULL would drop the value without a trace. +fn type_mismatch(col_name: &str, data_type: &DataType, value: &ColumnValue) -> CdcError { + CdcError::Iceberg(format!( + "column {col_name} is {data_type} but the source sent {} ({}); \ + the source field type changed and the value cannot be converted", + crate::schema::value::column_value_to_string(Some(value)).unwrap_or_default(), + crate::schema::value::column_value_type_name(value), + )) +} + +fn convert_values( + col_name: &str, + data_type: &DataType, + values: &[Option<&ColumnValue>], + convert: impl Fn(&ColumnValue) -> Option, +) -> Result>> { + values + .iter() + .map(|v| match v { + None | Some(ColumnValue::Null) | Some(ColumnValue::UnchangedToast) => Ok(None), + Some(value) => convert(value) + .map(Some) + .ok_or_else(|| type_mismatch(col_name, data_type, value)), + }) + .collect() +} + /// Build an Arrow array for a source column from the extracted rows. fn build_source_column( col_name: &str, @@ -88,73 +118,58 @@ fn build_source_column( match data_type { DataType::Int32 => { - let arr: Int32Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Int(n)) => Some(*n as i32), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Int(n) => i32::try_from(*n).ok(), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Int32Array::from(vals))) } DataType::Int64 => { - let arr: Int64Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Int(n)) => Some(*n), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Int(n) => Some(*n), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Int64Array::from(vals))) } DataType::Float32 => { - let arr: Float32Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Float(f)) => Some(f.into_inner() as f32), - Some(ColumnValue::Int(n)) => Some(*n as f32), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Float(f) => Some(f.into_inner() as f32), + ColumnValue::Int(n) => Some(*n as f32), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Float32Array::from(vals))) } DataType::Float64 => { - let arr: Float64Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Float(f)) => Some(f.into_inner()), - Some(ColumnValue::Int(n)) => Some(*n as f64), - Some(ColumnValue::Text(s)) => s.parse::().ok(), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Float(f) => Some(f.into_inner()), + ColumnValue::Int(n) => Some(*n as f64), + ColumnValue::Text(s) => s.parse::().ok(), + _ => None, + })?; + Ok(Arc::new(Float64Array::from(vals))) } DataType::Boolean => { - let arr: BooleanArray = values - .iter() - .map(|v| match v { - Some(ColumnValue::Bool(b)) => Some(*b), - Some(ColumnValue::Int(n)) => Some(*n != 0), - Some(ColumnValue::Text(s)) => Some(s == "t" || s == "true" || s == "1"), + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Bool(b) => Some(*b), + ColumnValue::Int(n) => Some(*n != 0), + ColumnValue::Text(s) => match s.as_str() { + "t" | "true" | "TRUE" | "1" => Some(true), + "f" | "false" | "FALSE" | "0" => Some(false), _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + }, + _ => None, + })?; + Ok(Arc::new(BooleanArray::from(vals))) } DataType::Timestamp(TimeUnit::Microsecond, tz) => { - let arr: TimestampMicrosecondArray = values - .iter() - .map(|v| match v { - Some(ColumnValue::Timestamp(s)) | Some(ColumnValue::Text(s)) => { - parse_timestamp_us(s) - } - _ => None, - }) - .collect(); + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Timestamp(s) | ColumnValue::Text(s) => parse_timestamp_us(s), + _ => None, + })?; + let arr = TimestampMicrosecondArray::from(vals); let arr = match tz { Some(tz) => arr.with_timezone(tz.as_ref()), None => arr, @@ -162,78 +177,66 @@ fn build_source_column( Ok(Arc::new(arr)) } DataType::Date32 => { - let arr: Date32Array = values - .iter() - .map(|v| match v { - Some(ColumnValue::Date(s)) | Some(ColumnValue::Text(s)) => { - chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") - .ok() - .map(|d| { - (d - chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()) - .num_days() as i32 - }) - } - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Date(s) | ColumnValue::Text(s) => { + chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") + .ok() + .map(|d| { + (d - chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()).num_days() + as i32 + }) + } + _ => None, + })?; + Ok(Arc::new(Date32Array::from(vals))) } DataType::Decimal128(precision, scale) => { let scale_i32 = *scale as i32; - let arr: Vec> = values - .iter() - .map(|v| match v { - Some(ColumnValue::Text(s)) => parse_decimal_i128(s, scale_i32), - Some(ColumnValue::Float(f)) => { - let multiplier = 10_f64.powi(scale_i32); - Some((f.into_inner() * multiplier).round() as i128) - } - Some(ColumnValue::Int(n)) => { - let multiplier = 10_i128.pow(scale_i32 as u32); - Some(*n as i128 * multiplier) - } - _ => None, - }) - .collect(); - let arr = Decimal128Array::from(arr) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Text(s) => parse_decimal_i128(s, scale_i32), + ColumnValue::Float(f) => { + let multiplier = 10_f64.powi(scale_i32); + Some((f.into_inner() * multiplier).round() as i128) + } + ColumnValue::Int(n) => { + let multiplier = 10_i128.pow(scale_i32 as u32); + Some(*n as i128 * multiplier) + } + _ => None, + })?; + let arr = Decimal128Array::from(vals) .with_precision_and_scale(*precision, *scale) .map_err(|e| CdcError::Iceberg(format!("decimal array: {e}")))?; Ok(Arc::new(arr)) } DataType::Time64(TimeUnit::Microsecond) => { - let arr: Time64MicrosecondArray = values - .iter() - .map(|v| match v { - Some(ColumnValue::Time(s)) | Some(ColumnValue::Text(s)) => parse_time_us(s), - _ => None, - }) - .collect(); - Ok(Arc::new(arr)) + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Time(s) | ColumnValue::Text(s) => parse_time_us(s), + _ => None, + })?; + Ok(Arc::new(Time64MicrosecondArray::from(vals))) } DataType::LargeBinary => { - let arr: LargeBinaryArray = values + let vals = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Bytes(b) => Some(b.clone()), + _ => None, + })?; + let arr: LargeBinaryArray = vals .iter() - .map(|v| match v { - Some(ColumnValue::Bytes(b)) => Some(b.as_slice()), - _ => None, - }) + .map(|v| v.as_ref().map(|b| b.as_slice())) .collect(); Ok(Arc::new(arr)) } DataType::FixedSizeBinary(16) => { - // UUID: parse from string or pass through bytes - let byte_values: Vec> = values - .iter() - .map(|v| match v { - Some(ColumnValue::Text(s)) => parse_uuid_bytes(s), - Some(ColumnValue::Bytes(b)) if b.len() == 16 => { - let mut arr = [0u8; 16]; - arr.copy_from_slice(b); - Some(arr) - } - _ => None, - }) - .collect(); + let byte_values = convert_values(col_name, data_type, &values, |v| match v { + ColumnValue::Text(s) => parse_uuid_bytes(s), + ColumnValue::Bytes(b) if b.len() == 16 => { + let mut arr = [0u8; 16]; + arr.copy_from_slice(b); + Some(arr) + } + _ => None, + })?; let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size( byte_values.iter().map(|v| v.as_ref().map(|b| b.as_slice())), 16, @@ -332,23 +335,53 @@ pub fn events_to_flattened_cdc_batch( .map_err(|e| CdcError::Iceberg(format!("flattened CDC record batch: {e}"))) } +/// Identity of a row within a batch, or None when the event carries no usable PK. +fn row_key(event: &CdcEvent) -> Option>> { + let row = event.new.as_ref()?; + if event.primary_key_columns.is_empty() { + return None; + } + Some( + event + .primary_key_columns + .iter() + .map(|c| crate::schema::value::column_value_to_string(row.get(c))) + .collect(), + ) +} + /// Convert CDC events into a data RecordBatch for replication mode (source columns only). /// /// Filters to Insert/Update/Snapshot events and uses `event.new` for row data. /// Returns None if no matching events exist. +/// +/// Only the last event per primary key survives: equality deletes never apply to +/// data files added by the same commit, so two versions of a key in one batch +/// would both remain in the table. pub fn events_to_replication_data_batch( events: &[CdcEvent], arrow_schema: &ArrowSchema, ) -> Result> { - let data_events: Vec<&CdcEvent> = events - .iter() - .filter(|e| { - matches!( - e.op, - ChangeOp::Insert | ChangeOp::Update | ChangeOp::Snapshot - ) - }) - .collect(); + let mut data_events: Vec<&CdcEvent> = Vec::new(); + let mut positions: HashMap>, usize> = HashMap::new(); + + for event in events.iter().filter(|e| { + matches!( + e.op, + ChangeOp::Insert | ChangeOp::Update | ChangeOp::Snapshot + ) + }) { + match row_key(event) { + Some(key) => match positions.get(&key) { + Some(&pos) => data_events[pos] = event, + None => { + positions.insert(key, data_events.len()); + data_events.push(event); + } + }, + None => data_events.push(event), + } + } if data_events.is_empty() { return Ok(None); @@ -966,6 +999,119 @@ mod tests { assert_eq!(pk.value(0), "[\"id\"]"); } + #[test] + fn test_repl_batch_rejects_value_of_wrong_type() { + let schema = ArrowSchema::new(vec![Field::new("age", DataType::Int64, true)]); + + let events = vec![CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([( + "age".into(), + ColumnValue::Text("thirty".into()), + )])), + old: None, + primary_key_columns: vec![], + }]; + + let err = events_to_replication_data_batch(&events, &schema).unwrap_err(); + assert!( + err.to_string().contains("thirty"), + "error should name the offending value: {err}" + ); + } + + #[test] + fn test_repl_batch_accepts_null_for_typed_column() { + let schema = ArrowSchema::new(vec![Field::new("age", DataType::Int64, true)]); + + let events = vec![CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([("age".into(), ColumnValue::Null)])), + old: None, + primary_key_columns: vec![], + }]; + + let batch = events_to_replication_data_batch(&events, &schema) + .unwrap() + .unwrap(); + assert!(batch.column(0).is_null(0)); + } + + #[test] + fn test_repl_batch_keeps_only_last_event_per_pk() { + let schema = ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ]); + + let event = |op, id: i64, name: &str| CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op, + new: Some(BTreeMap::from([ + ("id".into(), ColumnValue::Int(id)), + ("name".into(), ColumnValue::Text(name.into())), + ])), + old: None, + primary_key_columns: vec!["id".into()], + }; + + let events = vec![ + event(ChangeOp::Insert, 1, "first"), + event(ChangeOp::Update, 1, "second"), + event(ChangeOp::Update, 2, "other"), + event(ChangeOp::Update, 1, "third"), + ]; + + let batch = events_to_replication_data_batch(&events, &schema) + .unwrap() + .unwrap(); + + assert_eq!(batch.num_rows(), 2); + let names = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(names.value(0), "third"); + assert_eq!(names.value(1), "other"); + } + + #[test] + fn test_repl_batch_keeps_all_events_without_pk_columns() { + let schema = ArrowSchema::new(vec![Field::new("name", DataType::Utf8, true)]); + + let event = |name: &str| CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_000_000, + xid: 1, + table: test_table_id(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([( + "name".into(), + ColumnValue::Text(name.into()), + )])), + old: None, + primary_key_columns: vec![], + }; + + let batch = events_to_replication_data_batch(&[event("a"), event("b")], &schema) + .unwrap() + .unwrap(); + + assert_eq!(batch.num_rows(), 2); + } + // ── Type conversion tests (via replication data batch) ────────── #[test] diff --git a/src/sink/postgres/mod.rs b/src/sink/postgres/mod.rs index b785387..6540913 100644 --- a/src/sink/postgres/mod.rs +++ b/src/sink/postgres/mod.rs @@ -16,6 +16,7 @@ use schema::{ }; use crate::schema::value::column_value_to_string; +use crate::sink::postgres::schema::quote_ident; /// Cached information about a target table. struct TableInfo { @@ -78,7 +79,11 @@ impl PostgresSink { let target_table = format!("{}{}", self.config.table_prefix, source_table); // Clone to avoid borrowing self.config while we need &mut self later. let target_schema = self.config.schema.clone(); - let target_fqn = format!("\"{}\".\"{}\"", target_schema, target_table); + let target_fqn = format!( + "{}.{}", + quote_ident(&target_schema), + quote_ident(&target_table) + ); match self.mode { SinkMode::Cdc => { @@ -135,7 +140,9 @@ impl PostgresSink { .map(|col| { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; (col.name.clone(), sql_type) }) @@ -229,7 +236,9 @@ impl PostgresSink { .map(|col| { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; (col.name.clone(), sql_type) }) @@ -306,7 +315,9 @@ impl PostgresSink { for col in &source_columns { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; typed_cols.push((col.name.clone(), sql_type.clone())); typed_cols.push((format!("_old_{}", col.name), sql_type)); @@ -320,7 +331,9 @@ impl PostgresSink { for col in &source_columns { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; info_mut.columns.push(col.name.clone()); info_mut.column_types.insert(col.name.clone(), sql_type); @@ -362,12 +375,12 @@ impl PostgresSink { for col in &info.columns { let val = event.new.as_ref().and_then(|r| r.get(col)); - values.push(column_value_to_string(val)); + values.push(param_value(val)); } for col in &info.columns { let val = event.old.as_ref().and_then(|r| r.get(col)); - values.push(column_value_to_string(val)); + values.push(param_value(val)); } let params: Vec<&(dyn ToSql + Sync)> = @@ -425,7 +438,9 @@ impl PostgresSink { .map(|col| { let sql_type = match dialect { SourceDialect::Postgres => col.source_type.clone(), - SourceDialect::Mysql => canonical_to_pg_ddl(&col.canonical_type), + SourceDialect::Mysql | SourceDialect::Mongodb => { + canonical_to_pg_ddl(&col.canonical_type) + } }; (col.name.clone(), sql_type) }) @@ -486,7 +501,11 @@ impl PostgresSink { if !missing_from_events.is_empty() { // Confirm the drop by querying the source database - if let Some(source_conn) = self.source_connection.as_ref() { + if let Some(source_conn) = self + .source_connection + .as_ref() + .filter(|c| crate::schema::evolution::source_supports_column_drops(c)) + { let source_cols = crate::schema::discovery::fetch_columns(source_conn, schema, table) .await?; @@ -614,7 +633,7 @@ fn build_upsert_sql( pk_columns: &[String], column_types: &HashMap, ) -> String { - let col_list: Vec = columns.iter().map(|c| format!("\"{}\"", c)).collect(); + let col_list: Vec = columns.iter().map(|c| quote_ident(c)).collect(); let placeholders: Vec = columns .iter() .enumerate() @@ -629,10 +648,10 @@ fn build_upsert_sql( .filter(|(_, c)| !pk_columns.contains(c)) .map(|(i, c)| { let typ = column_types.get(c).map(|t| t.as_str()).unwrap_or("text"); - format!("\"{}\" = ${}::{}", c, i + 1, typ) + format!("{} = ${}::{}", quote_ident(c), i + 1, typ) }) .collect(); - let pk_list: Vec = pk_columns.iter().map(|c| format!("\"{}\"", c)).collect(); + let pk_list: Vec = pk_columns.iter().map(|c| quote_ident(c)).collect(); if update_sets.is_empty() { format!( @@ -668,7 +687,7 @@ fn build_delete_sql( .enumerate() .map(|(i, c)| { let typ = column_types.get(c).map(|t| t.as_str()).unwrap_or("text"); - format!("\"{}\" = ${}::{}", c, i + 1, typ) + format!("{} = ${}::{}", quote_ident(c), i + 1, typ) }) .collect(); format!( @@ -678,6 +697,15 @@ fn build_delete_sql( ) } +/// Params go over the wire as TEXT and are cast server-side, so bytes need the +/// `\x` hex prefix — bare hex is read as the escape format and stored literally. +fn param_value(value: Option<&ColumnValue>) -> Option { + match value { + Some(ColumnValue::Bytes(_)) => column_value_to_string(value).map(|hex| format!("\\x{hex}")), + _ => column_value_to_string(value), + } +} + /// UPSERT a row: INSERT ... ON CONFLICT (pk) DO UPDATE SET ... /// /// Uses `prepare_typed` with all `Type::TEXT` so tokio-postgres sends params @@ -690,10 +718,7 @@ async fn upsert_row( ) -> Result<()> { let cols = &info.columns; - let values: Vec> = cols - .iter() - .map(|c| crate::schema::value::column_value_to_string(row.get(c))) - .collect(); + let values: Vec> = cols.iter().map(|c| param_value(row.get(c))).collect(); let sql = build_upsert_sql(&info.target_fqn, cols, &info.pk_columns, &info.column_types); @@ -717,10 +742,7 @@ async fn delete_row( ) -> Result<()> { let pk_cols = &info.pk_columns; - let pk_values: Vec> = pk_cols - .iter() - .map(|c| crate::schema::value::column_value_to_string(row.get(c))) - .collect(); + let pk_values: Vec> = pk_cols.iter().map(|c| param_value(row.get(c))).collect(); let sql = build_delete_sql(&info.target_fqn, pk_cols, &info.column_types); @@ -748,6 +770,23 @@ fn connection_string(config: &PostgresSinkConfig) -> String { mod tests { use super::*; + #[test] + fn test_bytes_param_uses_hex_bytea_literal() { + assert_eq!( + param_value(Some(&ColumnValue::Bytes(vec![0x68, 0x00, 0xff]))), + Some("\\x6800ff".to_string()) + ); + } + + #[test] + fn test_non_bytes_params_are_unchanged() { + assert_eq!( + param_value(Some(&ColumnValue::Text("\\x41".into()))), + Some("\\x41".to_string()) + ); + assert_eq!(param_value(Some(&ColumnValue::Null)), None); + } + #[test] fn test_connection_string_with_password() { let config = PostgresSinkConfig { diff --git a/src/sink/postgres/schema.rs b/src/sink/postgres/schema.rs index 6f2079e..5323df1 100644 --- a/src/sink/postgres/schema.rs +++ b/src/sink/postgres/schema.rs @@ -41,9 +41,15 @@ pub fn canonical_to_pg_ddl(ct: &CanonicalType) -> String { } } +/// Quote an identifier. MongoDB field names may contain `"`, which would +/// otherwise terminate the quoted identifier and let the rest run as SQL. +pub fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + /// Build a fully qualified table name: "schema"."table" fn build_fqn(schema: &str, table: &str) -> String { - format!("\"{}\".\"{}\"", schema, table) + format!("{}.{}", quote_ident(schema), quote_ident(table)) } /// Build the DDL string for creating a CDC-mode table with flattened typed columns. @@ -66,11 +72,11 @@ fn build_cdc_table_ddl(schema: &str, table: &str, source_columns: &[(String, Str ]; for (name, typ) in source_columns { - col_defs.push(format!("\"{}\" {}", name, typ)); + col_defs.push(format!("{} {}", quote_ident(name), typ)); } for (name, typ) in source_columns { - col_defs.push(format!("\"_old_{}\" {}", name, typ)); + col_defs.push(format!("{} {}", quote_ident(&format!("_old_{name}")), typ)); } format!( @@ -112,14 +118,14 @@ fn build_cdc_insert_sql( let mut idx = 8; for col in source_columns { - col_names.push(format!("\"{}\"", col)); + col_names.push(quote_ident(col)); let typ = column_types.get(col).map(|t| t.as_str()).unwrap_or("text"); placeholders.push(format!("${}::{}", idx, typ)); idx += 1; } for col in source_columns { - col_names.push(format!("\"_old_{}\"", col)); + col_names.push(quote_ident(&format!("_old_{col}"))); let typ = column_types.get(col).map(|t| t.as_str()).unwrap_or("text"); placeholders.push(format!("${}::{}", idx, typ)); idx += 1; @@ -145,9 +151,9 @@ fn build_replication_table_ddl( let fqn = build_fqn(schema, table); let col_defs: Vec = columns .iter() - .map(|(name, typ)| format!("\"{}\" {}", name, typ)) + .map(|(name, typ)| format!("{} {}", quote_ident(name), typ)) .collect(); - let pk_list: Vec = pk_columns.iter().map(|c| format!("\"{}\"", c)).collect(); + let pk_list: Vec = pk_columns.iter().map(|c| quote_ident(c)).collect(); format!( "CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))", fqn, @@ -163,7 +169,7 @@ fn build_add_columns_ddl(schema: &str, table: &str, new_columns: &[(String, Stri let fqn = build_fqn(schema, table); let alter_clauses: Vec = new_columns .iter() - .map(|(name, typ)| format!("ADD COLUMN IF NOT EXISTS \"{}\" {}", name, typ)) + .map(|(name, typ)| format!("ADD COLUMN IF NOT EXISTS {} {}", quote_ident(name), typ)) .collect(); format!("ALTER TABLE {} {}", fqn, alter_clauses.join(", ")) } @@ -175,7 +181,7 @@ fn build_drop_columns_ddl(schema: &str, table: &str, columns: &[String]) -> Stri let fqn = build_fqn(schema, table); let alter_clauses: Vec = columns .iter() - .map(|name| format!("DROP COLUMN IF EXISTS \"{}\"", name)) + .map(|name| format!("DROP COLUMN IF EXISTS {}", quote_ident(name))) .collect(); format!("ALTER TABLE {} {}", fqn, alter_clauses.join(", ")) } @@ -330,6 +336,27 @@ pub async fn add_columns( mod tests { use super::*; + #[test] + fn test_quote_ident_escapes_embedded_quotes() { + assert_eq!(quote_ident("bad\"name"), "\"bad\"\"name\""); + } + + #[test] + fn test_ddl_with_hostile_field_name_stays_one_identifier() { + let ddl = build_cdc_table_ddl( + "public", + "weird", + &[( + "x\" text); DROP TABLE t; --".to_string(), + "text".to_string(), + )], + ); + assert!( + ddl.contains("\"x\"\" text); DROP TABLE t; --\" text"), + "field name must stay inside one quoted identifier: {ddl}" + ); + } + #[test] fn test_build_fqn() { assert_eq!(build_fqn("public", "users"), "\"public\".\"users\""); diff --git a/src/source/mod.rs b/src/source/mod.rs index bd46693..239f61a 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -1,3 +1,4 @@ +pub mod mongodb; pub mod mysql; pub mod postgres; diff --git a/src/source/mongodb/bson_mapping.rs b/src/source/mongodb/bson_mapping.rs new file mode 100644 index 0000000..6ee5575 --- /dev/null +++ b/src/source/mongodb/bson_mapping.rs @@ -0,0 +1,398 @@ +use std::collections::BTreeMap; + +use mongodb::bson::{Bson, Document}; + +use crate::event::{ColumnValue, Row}; + +/// Convert a BSON value to a `ColumnValue`. +/// +/// Top-level fields are flattened: nested documents and arrays are serialized +/// as JSON text rather than recursed into. +pub fn bson_to_column_value(bson: &Bson) -> ColumnValue { + match bson { + Bson::Null => ColumnValue::Null, + Bson::Boolean(b) => ColumnValue::Bool(*b), + Bson::Int32(i) => ColumnValue::Int(*i as i64), + Bson::Int64(i) => ColumnValue::Int(*i), + Bson::Double(d) => ColumnValue::float(*d), + Bson::String(s) => ColumnValue::Text(s.clone()), + Bson::ObjectId(oid) => ColumnValue::Text(oid.to_hex()), + Bson::DateTime(dt) => { + // `from_timestamp_millis` handles negative (pre-1970) values. + let ts = chrono::DateTime::from_timestamp_millis(dt.timestamp_millis()) + .unwrap_or_default() + .to_rfc3339(); + ColumnValue::Timestamp(ts) + } + Bson::Binary(bin) => ColumnValue::Bytes(bin.bytes.clone()), + Bson::Decimal128(d) => ColumnValue::Text(d.to_string()), + Bson::Timestamp(ts) => { + // BSON Timestamp is (seconds, increment) — used internally by oplog. + // Format as seconds since epoch. + ColumnValue::Text(format!("{}:{}", ts.time, ts.increment)) + } + Bson::RegularExpression(re) => ColumnValue::Text(format!("/{}/{}", re.pattern, re.options)), + // Nested documents and arrays → JSON text + Bson::Document(_) | Bson::Array(_) => { + let json = serde_json::to_string(bson).unwrap_or_default(); + ColumnValue::Text(json) + } + // Remaining BSON types → text representation + _ => { + let json = serde_json::to_string(bson).unwrap_or_default(); + ColumnValue::Text(json) + } + } +} + +/// Convert a BSON document to a CDC `Row` (BTreeMap of field → ColumnValue). +pub fn document_to_row(doc: &Document) -> Row { + let mut row: BTreeMap = BTreeMap::new(); + for (key, value) in doc { + row.insert(key.clone(), bson_to_column_value(value)); + } + row +} + +/// Return the BSON type name string for schema inference. +pub fn bson_type_string(bson: &Bson) -> &'static str { + match bson { + Bson::Null => "null", + Bson::Boolean(_) => "bool", + Bson::Int32(_) => "int", + Bson::Int64(_) => "long", + Bson::Double(_) => "double", + Bson::String(_) => "string", + Bson::ObjectId(_) => "objectid", + Bson::DateTime(_) => "date", + Bson::Binary(_) => "bindata", + Bson::Decimal128(_) => "decimal", + Bson::Document(_) => "object", + Bson::Array(_) => "array", + Bson::Timestamp(_) => "string", + Bson::RegularExpression(_) => "string", + _ => "string", + } +} + +/// Merge two BSON type names observed for the same field into one that holds +/// both: nulls carry no type, numerics widen, anything else falls back to text. +pub fn merge_bson_types(existing: &str, incoming: &str) -> String { + if existing == incoming { + return existing.to_string(); + } + match (existing, incoming) { + ("null", other) | (other, "null") => other.to_string(), + ("int", "long") | ("long", "int") => "long".to_string(), + ("int", "double") | ("double", "int") => "double".to_string(), + ("long", "double") | ("double", "long") => "double".to_string(), + ("object", "array") | ("array", "object") => "object".to_string(), + _ => "string".to_string(), + } +} + +pub fn merge_document_field_types(field_types: &mut BTreeMap, doc: &Document) { + for (key, value) in doc { + let incoming = bson_type_string(value); + match field_types.get_mut(key) { + Some(existing) => { + *existing = merge_bson_types(existing, incoming); + } + None => { + field_types.insert(key.clone(), incoming.to_string()); + } + } + } +} + +pub fn infer_schema_from_documents(docs: &[Document]) -> Vec<(String, String)> { + let mut field_types: BTreeMap = BTreeMap::new(); + for doc in docs { + merge_document_field_types(&mut field_types, doc); + } + field_types.into_iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use mongodb::bson::{doc, oid::ObjectId, Binary, Decimal128, Regex}; + + #[test] + fn test_bson_to_column_value_primitives() { + let cases: Vec<(Bson, ColumnValue)> = vec![ + (Bson::Null, ColumnValue::Null), + (Bson::Boolean(true), ColumnValue::Bool(true)), + (Bson::Boolean(false), ColumnValue::Bool(false)), + (Bson::Int32(42), ColumnValue::Int(42)), + (Bson::Int32(-1), ColumnValue::Int(-1)), + (Bson::Int64(i64::MAX), ColumnValue::Int(i64::MAX)), + (Bson::Double(3.14), ColumnValue::float(3.14)), + ( + Bson::String("hello".into()), + ColumnValue::Text("hello".into()), + ), + ]; + for (bson, expected) in cases { + assert_eq!(bson_to_column_value(&bson), expected, "for {bson:?}"); + } + } + + #[test] + fn test_bson_to_column_value_objectid() { + let oid = ObjectId::parse_str("507f1f77bcf86cd799439011").unwrap(); + let result = bson_to_column_value(&Bson::ObjectId(oid)); + assert_eq!(result, ColumnValue::Text("507f1f77bcf86cd799439011".into())); + } + + #[test] + fn test_bson_to_column_value_binary() { + let bin = Binary { + subtype: mongodb::bson::spec::BinarySubtype::Generic, + bytes: vec![0xDE, 0xAD], + }; + let result = bson_to_column_value(&Bson::Binary(bin)); + assert_eq!(result, ColumnValue::Bytes(vec![0xDE, 0xAD])); + } + + #[test] + fn test_bson_to_column_value_datetime() { + let dt = mongodb::bson::DateTime::from_millis(1700000000000); + let result = bson_to_column_value(&Bson::DateTime(dt)); + match result { + ColumnValue::Timestamp(s) => { + assert!(s.contains("2023"), "expected 2023 in timestamp: {s}"); + } + other => panic!("expected Timestamp, got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_decimal128() { + let d = Decimal128::from_bytes([0; 16]); + let result = bson_to_column_value(&Bson::Decimal128(d)); + match result { + ColumnValue::Text(_) => {} // Decimal128 stored as text + other => panic!("expected Text, got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_nested_doc() { + let bson = Bson::Document(doc! { "nested": "value" }); + let result = bson_to_column_value(&bson); + match result { + ColumnValue::Text(json) => { + assert!(json.contains("nested"), "expected JSON: {json}"); + } + other => panic!("expected Text (JSON), got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_array() { + let bson = Bson::Array(vec![Bson::Int32(1), Bson::Int32(2)]); + let result = bson_to_column_value(&bson); + match result { + ColumnValue::Text(json) => { + assert_eq!(json, "[1,2]"); + } + other => panic!("expected Text (JSON array), got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_regex() { + let re = Regex { + pattern: "^abc".into(), + options: "i".into(), + }; + let result = bson_to_column_value(&Bson::RegularExpression(re)); + assert_eq!(result, ColumnValue::Text("/^abc/i".into())); + } + + #[test] + fn test_document_to_row() { + let doc = doc! { + "_id": ObjectId::new(), + "name": "Alice", + "age": 30, + "active": true, + "score": 95.5, + }; + let row = document_to_row(&doc); + assert_eq!(row.len(), 5); + assert!(matches!(row.get("name"), Some(ColumnValue::Text(s)) if s == "Alice")); + assert_eq!(row.get("age"), Some(&ColumnValue::Int(30))); + assert_eq!(row.get("active"), Some(&ColumnValue::Bool(true))); + assert_eq!(row.get("score"), Some(&ColumnValue::float(95.5))); + } + + #[test] + fn test_document_to_row_empty() { + let doc = doc! {}; + let row = document_to_row(&doc); + assert!(row.is_empty()); + } + + #[test] + fn test_bson_type_string() { + let cases: Vec<(Bson, &str)> = vec![ + (Bson::Null, "null"), + (Bson::Boolean(true), "bool"), + (Bson::Int32(0), "int"), + (Bson::Int64(0), "long"), + (Bson::Double(0.0), "double"), + (Bson::String("".into()), "string"), + (Bson::ObjectId(ObjectId::new()), "objectid"), + (Bson::DateTime(mongodb::bson::DateTime::now()), "date"), + ( + Bson::Binary(Binary { + subtype: mongodb::bson::spec::BinarySubtype::Generic, + bytes: vec![], + }), + "bindata", + ), + (Bson::Document(doc! {}), "object"), + (Bson::Array(vec![]), "array"), + (Bson::Decimal128(Decimal128::from_bytes([0; 16])), "decimal"), + ]; + for (bson, expected) in cases { + assert_eq!(bson_type_string(&bson), expected, "for {bson:?}"); + } + } + + #[test] + fn test_infer_schema_from_documents() { + let docs = vec![ + doc! { "_id": ObjectId::new(), "name": "Alice", "age": 30 }, + doc! { "_id": ObjectId::new(), "name": "Bob", "email": "bob@example.com" }, + ]; + let schema = infer_schema_from_documents(&docs); + let names: Vec<&str> = schema.iter().map(|(n, _)| n.as_str()).collect(); + // BTreeMap gives sorted order + assert_eq!(names, vec!["_id", "age", "email", "name"]); + + // Types should be inferred from first seen + let type_map: BTreeMap<&str, &str> = schema + .iter() + .map(|(n, t)| (n.as_str(), t.as_str())) + .collect(); + assert_eq!(type_map["_id"], "objectid"); + assert_eq!(type_map["name"], "string"); + assert_eq!(type_map["age"], "int"); + assert_eq!(type_map["email"], "string"); + } + + #[test] + fn test_infer_schema_empty() { + let schema = infer_schema_from_documents(&[]); + assert!(schema.is_empty()); + } + + #[test] + fn test_bson_to_column_value_timestamp() { + let ts = mongodb::bson::Timestamp { + time: 1700000000, + increment: 5, + }; + let result = bson_to_column_value(&Bson::Timestamp(ts)); + assert_eq!(result, ColumnValue::Text("1700000000:5".into())); + } + + #[test] + fn test_bson_to_column_value_datetime_before_epoch() { + // 1969-12-31T23:59:59.500Z — negative millis with a fractional part. + let dt = mongodb::bson::DateTime::from_millis(-500); + let result = bson_to_column_value(&Bson::DateTime(dt)); + match result { + ColumnValue::Timestamp(s) => { + assert!( + s.starts_with("1969-12-31T23:59:59.5"), + "expected a pre-epoch timestamp, got {s}" + ); + } + other => panic!("expected Timestamp, got {other:?}"), + } + } + + #[test] + fn test_bson_to_column_value_min_max_key_fallback() { + for bson in [Bson::MinKey, Bson::MaxKey, Bson::Undefined] { + assert!( + matches!(bson_to_column_value(&bson), ColumnValue::Text(_)), + "expected Text fallback for {bson:?}" + ); + } + } + + #[test] + fn test_bson_to_column_value_empty_binary() { + let bin = Binary { + subtype: mongodb::bson::spec::BinarySubtype::Generic, + bytes: vec![], + }; + assert_eq!( + bson_to_column_value(&Bson::Binary(bin)), + ColumnValue::Bytes(vec![]) + ); + } + + #[test] + fn test_merge_bson_types_null_is_ignored() { + assert_eq!(merge_bson_types("null", "int"), "int"); + assert_eq!(merge_bson_types("int", "null"), "int"); + assert_eq!(merge_bson_types("null", "null"), "null"); + } + + #[test] + fn test_merge_bson_types_numeric_widening() { + let cases = vec![ + (("int", "long"), "long"), + (("long", "int"), "long"), + (("int", "double"), "double"), + (("double", "long"), "double"), + ]; + for ((a, b), expected) in cases { + assert_eq!(merge_bson_types(a, b), expected, "for ({a}, {b})"); + } + } + + #[test] + fn test_merge_bson_types_conflict_falls_back_to_string() { + assert_eq!(merge_bson_types("int", "bool"), "string"); + assert_eq!(merge_bson_types("date", "objectid"), "string"); + assert_eq!(merge_bson_types("object", "array"), "object"); + } + + #[test] + fn test_merge_bson_types_identical() { + assert_eq!(merge_bson_types("string", "string"), "string"); + } + + #[test] + fn test_infer_schema_ignores_null_when_typing_field() { + let docs = vec![doc! { "score": Bson::Null }, doc! { "score": 42i32 }]; + let schema = infer_schema_from_documents(&docs); + assert_eq!(schema, vec![("score".to_string(), "int".to_string())]); + } + + #[test] + fn test_infer_schema_widens_conflicting_types() { + let docs = vec![ + doc! { "id": 1i32, "v": 1i32 }, + doc! { "id": "a", "v": 2i64 }, + ]; + let type_map: BTreeMap = + infer_schema_from_documents(&docs).into_iter().collect(); + assert_eq!(type_map["id"], "string"); // int + string → text column + assert_eq!(type_map["v"], "long"); // int + long → long + } + + #[test] + fn test_infer_schema_all_null_field_stays_null() { + let docs = vec![doc! { "x": Bson::Null }, doc! { "x": Bson::Null }]; + let schema = infer_schema_from_documents(&docs); + assert_eq!(schema, vec![("x".to_string(), "null".to_string())]); + } +} diff --git a/src/source/mongodb/change_stream.rs b/src/source/mongodb/change_stream.rs new file mode 100644 index 0000000..95ea282 --- /dev/null +++ b/src/source/mongodb/change_stream.rs @@ -0,0 +1,250 @@ +use std::time::Duration; + +use mongodb::bson::{doc, Document, Timestamp}; +use mongodb::change_stream::event::{ChangeStreamEvent, ResumeToken}; +use mongodb::change_stream::ChangeStream; +use mongodb::options::FullDocumentBeforeChangeType; +use mongodb::options::FullDocumentType; +use mongodb::Client; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use crate::error::{CdcError, Result}; +use crate::source::SourceEvent; + +use super::event_converter::{change_event_to_cdc_event, ConvertResult}; +use super::offset::MongoOffset; +use super::{send_event, SendOutcome}; + +const MAX_EVENTS_PER_COMMIT: usize = 1000; + +/// Server-side wait for new events; bounds partial-batch commit latency. +const MAX_AWAIT_TIME: Duration = Duration::from_millis(500); + +/// Determines where to start the change stream from. +pub enum ResumePoint { + /// Start after a snapshot at the given cluster time. + AfterSnapshot { snapshot_time: Timestamp }, + /// Resume from a previously saved resume token. + Token(ResumeToken), +} + +/// Uses `updateLookup` for the post-image and `whenAvailable` for the +/// pre-image; `collections` becomes a `$match` stage. +pub async fn open_change_stream( + client: &Client, + database: &str, + collections: &[String], + resume_point: &ResumePoint, +) -> Result>> { + let db = client.database(database); + + let pipeline: Vec = if collections.is_empty() { + vec![] + } else { + vec![doc! { + "$match": { + "ns.coll": { "$in": collections } + } + }] + }; + + let mut builder = db + .watch() + .pipeline(pipeline) + .full_document(FullDocumentType::UpdateLookup) + .full_document_before_change(FullDocumentBeforeChangeType::WhenAvailable) + .max_await_time(MAX_AWAIT_TIME); + + match resume_point { + ResumePoint::AfterSnapshot { snapshot_time } => { + builder = builder.start_at_operation_time(*snapshot_time); + } + ResumePoint::Token(token) => { + builder = builder.resume_after(token.clone()); + } + } + + builder + .await + .map_err(|e| stream_error("failed to open change stream", e.to_string())) +} + +/// An update to a large document carries both a post- and a pre-image, so the +/// event can exceed the 16 MB BSON limit even though each document is legal. +/// It then replays on every restart, which the driver's own text does not hint at. +fn stream_error(context: &str, detail: String) -> CdcError { + let mut msg = format!("{context}: {detail}"); + if detail.contains("BSONObjectTooLarge") { + msg.push_str( + " — the change event exceeded MongoDB's 16 MB limit, most likely an update to a \ + large document with changeStreamPreAndPostImages enabled. The event replays on \ + every restart: disable pre-images for the collection and re-snapshot to get past it.", + ); + } + CdcError::Mongodb(msg) +} + +/// True if the start position has aged out of the oplog — it can never become +/// valid again, so the only way forward is a fresh snapshot. +pub fn is_resume_point_lost(err: &CdcError) -> bool { + let msg = err.to_string(); + msg.contains("ChangeStreamHistoryLost") + || msg.contains("resume point may no longer be in the oplog") + || msg.contains("as the resume point may no longer be in the oplog") + || msg.contains("resume of change stream was not possible") +} + +/// Commits the latest resume token once a server batch is drained (or after +/// `MAX_EVENTS_PER_COMMIT` events), not once per document. +pub async fn pump_change_stream( + stream: &mut ChangeStream>, + database: &str, + sender: &mpsc::Sender, + shutdown: &CancellationToken, +) -> Result<()> { + tracing::info!(database, "change stream started"); + + let mut pending_offset: Option = None; + let mut pending_events: usize = 0; + + loop { + let next = tokio::select! { + _ = shutdown.cancelled() => { + tracing::info!("shutdown received, stopping change stream"); + commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown).await?; + return Ok(()); + } + // At most one `getMore`: an empty result means the batch is drained. + next = stream.next_if_any() => next, + }; + + match next { + Ok(Some(event)) => { + // Skipped events still advance the position — except + // `invalidate`, which the server refuses in `resumeAfter`. + let is_invalidate = matches!( + event.operation_type, + mongodb::change_stream::event::OperationType::Invalidate + ); + let offset = if is_invalidate { + None + } else { + Some(MongoOffset::Token(event.id.clone()).to_json()?) + }; + + match change_event_to_cdc_event(&event, database)? { + ConvertResult::Event(cdc_event) => { + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Change(cdc_event), shutdown).await? + { + return Ok(()); + } + } + ConvertResult::Skip => { + tracing::debug!(op_type = ?event.operation_type, "skipping unsupported change event"); + } + } + + if let Some(offset) = offset { + pending_offset = Some(offset); + } + pending_events += 1; + + if pending_events >= MAX_EVENTS_PER_COMMIT { + commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown) + .await?; + } + } + Ok(None) => { + // With no event token (idle or fully filtered stream), the + // stream's own token keeps the offset from ageing out. + if pending_offset.is_none() && stream.is_alive() { + if let Some(token) = stream.resume_token() { + pending_offset = Some(MongoOffset::Token(token).to_json()?); + } + } + + commit_pending(sender, &mut pending_offset, &mut pending_events, shutdown).await?; + + if !stream.is_alive() { + return Err(CdcError::Mongodb( + "change stream closed by the server (the database or a watched collection \ + was likely dropped or renamed)" + .into(), + )); + } + } + Err(e) => { + return Err(stream_error("change stream error", e.to_string())); + } + } + } +} + +async fn commit_pending( + sender: &mpsc::Sender, + pending_offset: &mut Option, + pending_events: &mut usize, + shutdown: &CancellationToken, +) -> Result<()> { + if let Some(offset) = pending_offset.take() { + send_event(sender, SourceEvent::Commit { offset }, shutdown).await?; + } + *pending_events = 0; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lost_resume_point_errors_are_recognized() { + let cases = [ + "failed to open change stream: Kind: Command failed: Error code 286 \ + (ChangeStreamHistoryLost)", + "failed to open change stream: resume of change stream was not possible, as the \ + resume point may no longer be in the oplog", + ]; + for msg in cases { + assert!( + is_resume_point_lost(&CdcError::Mongodb(msg.into())), + "expected a lost resume point for: {msg}" + ); + } + } + + #[test] + fn test_oversized_event_error_explains_pre_images() { + let err = stream_error( + "change stream error", + "Error code 10334 (BSONObjectTooLarge): Size 28312133 exceeds maximum 16793600".into(), + ); + assert!(err.to_string().contains("pre-images"), "{err}"); + } + + #[test] + fn test_unrelated_errors_get_no_extra_hint() { + let err = stream_error("change stream error", "connection refused".into()); + assert_eq!( + err.to_string(), + "mongodb error: change stream error: connection refused" + ); + } + + #[test] + fn test_other_errors_are_not_treated_as_lost_resume_points() { + let cases = [ + "failed to open change stream: connection refused", + "change stream error: not authorized on demo to execute command", + "failed to parse stored offset", + ]; + for msg in cases { + assert!( + !is_resume_point_lost(&CdcError::Mongodb(msg.into())), + "did not expect a lost resume point for: {msg}" + ); + } + } +} diff --git a/src/source/mongodb/event_converter.rs b/src/source/mongodb/event_converter.rs new file mode 100644 index 0000000..763992f --- /dev/null +++ b/src/source/mongodb/event_converter.rs @@ -0,0 +1,353 @@ +use mongodb::bson::Document; +use mongodb::change_stream::event::ChangeStreamEvent; + +use crate::error::Result; +use crate::event::{CdcEvent, ChangeOp, Lsn, TableId}; + +use super::bson_mapping::document_to_row; + +/// Result of converting a change stream event. +pub enum ConvertResult { + /// A CDC event to send downstream. + Event(CdcEvent), + /// Unsupported operation, or a CRUD event with no usable row. + Skip, +} + +/// Convert a MongoDB change stream event to a `CdcEvent`. +/// +/// Skips unsupported operations (drop, rename, etc.). `old` comes from the +/// pre-image, falling back to `documentKey` so update/delete always carry `_id`. +pub fn change_event_to_cdc_event( + event: &ChangeStreamEvent, + database: &str, +) -> Result { + let op = match event.operation_type { + mongodb::change_stream::event::OperationType::Insert => ChangeOp::Insert, + mongodb::change_stream::event::OperationType::Update + | mongodb::change_stream::event::OperationType::Replace => ChangeOp::Update, + mongodb::change_stream::event::OperationType::Delete => ChangeOp::Delete, + _ => return Ok(ConvertResult::Skip), + }; + + let collection = event + .ns + .as_ref() + .and_then(|ns| ns.coll.as_deref()) + .unwrap_or("unknown"); + + // Wall time is millisecond resolution; cluster time only seconds. + let timestamp_us = event + .wall_time + .map(|wt| wt.timestamp_millis() * 1000) + .or_else(|| event.cluster_time.map(|ts| ts.time as i64 * 1_000_000)) + .unwrap_or(0); + + let document_key_row = event.document_key.as_ref().map(document_to_row); + + let new = event.full_document.as_ref().map(document_to_row); + let old = match op { + ChangeOp::Update | ChangeOp::Delete => event + .full_document_before_change + .as_ref() + .map(document_to_row) + .or_else(|| document_key_row.clone()), + _ => None, + }; + + // No full document: removed before `updateLookup` resolved it. The later + // delete event covers it, so skip rather than emit a row-less change. + if matches!(op, ChangeOp::Insert | ChangeOp::Update) && new.is_none() { + tracing::warn!( + collection, + op_type = ?event.operation_type, + "change event has no full document (document removed before lookup), skipping" + ); + return Ok(ConvertResult::Skip); + } + + if op == ChangeOp::Delete && old.is_none() { + tracing::warn!( + collection, + "delete event has neither pre-image nor documentKey, skipping" + ); + return Ok(ConvertResult::Skip); + } + + let cdc_event = CdcEvent { + lsn: Lsn(0), + timestamp_us, + xid: 0, + table: TableId { + schema: database.to_string(), + name: collection.to_string(), + oid: 0, + }, + op, + new, + old, + primary_key_columns: vec!["_id".to_string()], + }; + + Ok(ConvertResult::Event(cdc_event)) +} + +/// Convert a snapshot document to a `CdcEvent`. +pub fn snapshot_doc_to_cdc_event(doc: &Document, database: &str, collection: &str) -> CdcEvent { + let row = document_to_row(doc); + CdcEvent { + lsn: Lsn(0), + timestamp_us: chrono::Utc::now().timestamp_micros(), + xid: 0, + table: TableId { + schema: database.to_string(), + name: collection.to_string(), + oid: 0, + }, + op: ChangeOp::Snapshot, + new: Some(row), + old: None, + primary_key_columns: vec!["_id".to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::ColumnValue; + use mongodb::bson::{doc, Timestamp}; + + /// Build a `ChangeStreamEvent` from a raw change event document, the same + /// way the driver deserializes events off the wire. + fn event_from_doc(doc: Document) -> ChangeStreamEvent { + mongodb::bson::from_document(doc).expect("failed to deserialize change event") + } + + /// A minimal change event document with the given operation type. + fn base_event(op_type: &str, coll: &str) -> Document { + doc! { + "_id": { "_data": "8265F1A2B3000000012B" }, + "operationType": op_type, + "ns": { "db": "mydb", "coll": coll }, + "clusterTime": Timestamp { time: 1_700_000_000, increment: 1 }, + } + } + + fn expect_event(result: ConvertResult) -> CdcEvent { + match result { + ConvertResult::Event(e) => e, + ConvertResult::Skip => panic!("expected Event, got Skip"), + } + } + + #[test] + fn test_insert_event_maps_to_insert() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Alice" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Insert); + assert_eq!(event.table.schema, "mydb"); + assert_eq!(event.table.name, "users"); + assert_eq!( + event.new.as_ref().and_then(|r| r.get("name")), + Some(&ColumnValue::Text("Alice".into())) + ); + assert!(event.old.is_none(), "insert must not carry an old row"); + } + + #[test] + fn test_update_event_maps_to_update_with_full_document() { + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Alice2" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Update); + assert_eq!( + event.new.as_ref().and_then(|r| r.get("name")), + Some(&ColumnValue::Text("Alice2".into())) + ); + } + + #[test] + fn test_replace_event_maps_to_update() { + let mut raw = base_event("replace", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Replaced" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Update); + } + + #[test] + fn test_update_without_pre_image_uses_document_key_as_old() { + // Pre-images are disabled by default in MongoDB; sinks still need the + // primary key to target the row being replaced. + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "Alice2" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + let old = event + .old + .expect("update should carry an old row with the pk"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("u1".into()))); + } + + #[test] + fn test_update_prefers_pre_image_over_document_key() { + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1", "name": "new" }); + raw.insert( + "fullDocumentBeforeChange", + doc! { "_id": "u1", "name": "old" }, + ); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + let old = event.old.expect("update should carry a pre-image"); + assert_eq!(old.get("name"), Some(&ColumnValue::Text("old".into()))); + } + + #[test] + fn test_delete_without_pre_image_uses_document_key_as_old() { + let mut raw = base_event("delete", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.op, ChangeOp::Delete); + assert!(event.new.is_none()); + let old = event.old.expect("delete must carry an old row with the pk"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("u1".into()))); + } + + #[test] + fn test_delete_with_pre_image_carries_full_old_row() { + let mut raw = base_event("delete", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert( + "fullDocumentBeforeChange", + doc! { "_id": "u1", "name": "Alice" }, + ); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + let old = event.old.unwrap(); + assert_eq!(old.get("name"), Some(&ColumnValue::Text("Alice".into()))); + } + + #[test] + fn test_delete_without_document_key_is_skipped() { + let raw = base_event("delete", "users"); + + let result = change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap(); + + assert!( + matches!(result, ConvertResult::Skip), + "a delete with no way to identify the row must be skipped" + ); + } + + #[test] + fn test_update_without_full_document_is_skipped() { + // The document was deleted before `updateLookup` could resolve it. + let mut raw = base_event("update", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + + let result = change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap(); + + assert!(matches!(result, ConvertResult::Skip)); + } + + #[test] + fn test_unsupported_operation_types_are_skipped() { + for op_type in ["drop", "rename", "dropDatabase", "invalidate"] { + let raw = base_event(op_type, "users"); + let result = change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap(); + assert!( + matches!(result, ConvertResult::Skip), + "expected {op_type} to be skipped" + ); + } + } + + #[test] + fn test_missing_namespace_falls_back_to_unknown_collection() { + let mut raw = base_event("insert", "users"); + raw.remove("ns"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.table.name, "unknown"); + } + + #[test] + fn test_wall_time_is_preferred_over_cluster_time() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + // Wall time has millisecond resolution; cluster time only seconds. + raw.insert( + "wallTime", + mongodb::bson::DateTime::from_millis(1_700_000_000_123), + ); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.timestamp_us, 1_700_000_000_123 * 1000); + } + + #[test] + fn test_cluster_time_used_when_wall_time_missing() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.timestamp_us, 1_700_000_000 * 1_000_000); + } + + #[test] + fn test_primary_key_is_always_id() { + let mut raw = base_event("insert", "users"); + raw.insert("documentKey", doc! { "_id": "u1" }); + raw.insert("fullDocument", doc! { "_id": "u1" }); + + let event = expect_event(change_event_to_cdc_event(&event_from_doc(raw), "mydb").unwrap()); + + assert_eq!(event.primary_key_columns, vec!["_id".to_string()]); + } + + #[test] + fn test_snapshot_doc_to_cdc_event() { + let doc = doc! { + "_id": "abc123", + "name": "Alice", + "age": 30, + }; + let event = snapshot_doc_to_cdc_event(&doc, "mydb", "users"); + assert_eq!(event.op, ChangeOp::Snapshot); + assert_eq!(event.table.schema, "mydb"); + assert_eq!(event.table.name, "users"); + assert_eq!(event.primary_key_columns, vec!["_id"]); + assert!(event.new.is_some()); + assert!(event.old.is_none()); + + let row = event.new.unwrap(); + assert!( + matches!(row.get("name"), Some(crate::event::ColumnValue::Text(s)) if s == "Alice") + ); + assert_eq!(row.get("age"), Some(&crate::event::ColumnValue::Int(30))); + } +} diff --git a/src/source/mongodb/mod.rs b/src/source/mongodb/mod.rs new file mode 100644 index 0000000..d020d1b --- /dev/null +++ b/src/source/mongodb/mod.rs @@ -0,0 +1,222 @@ +pub mod bson_mapping; +pub mod change_stream; +pub mod event_converter; +pub mod offset; +pub mod snapshot; + +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use crate::config::MongodbSourceConfig; +use crate::error::{CdcError, Result}; +use crate::offset::OffsetStore; +use crate::source::Source; + +use self::change_stream::{ + is_resume_point_lost, open_change_stream, pump_change_stream, ResumePoint, +}; +use self::offset::MongoOffset; +use self::snapshot::perform_snapshot; + +/// Fail fast on a configured collection that does not exist — MongoDB answers +/// reads on one with empty results, so a typo would silently replicate nothing. +async fn validate_collections( + client: &mongodb::Client, + database: &str, + collections: &[String], +) -> Result<()> { + if collections.is_empty() { + return Ok(()); + } + + let existing = client + .database(database) + .list_collection_names() + .await + .map_err(|e| CdcError::Mongodb(format!("failed to list collections: {e}")))?; + + let missing: Vec<&str> = collections + .iter() + .map(|c| c.as_str()) + .filter(|c| !existing.iter().any(|e| e == c)) + .collect(); + + if !missing.is_empty() { + return Err(CdcError::Mongodb(format!( + "configured collection(s) not found in database {database}: {}. Available: {}", + missing.join(", "), + if existing.is_empty() { + "(none)".to_string() + } else { + existing.join(", ") + } + ))); + } + + Ok(()) +} + +pub(crate) enum SendOutcome { + Sent, + /// The pipeline stopped receiving while a shutdown was in flight. + ShuttingDown, +} + +/// Send an event to the pipeline. The receive loop exits as soon as the +/// shutdown token fires, so a closed channel then is a clean stop, not an error. +pub(crate) async fn send_event( + sender: &mpsc::Sender, + event: crate::source::SourceEvent, + shutdown: &CancellationToken, +) -> Result { + match sender.send(event).await { + Ok(()) => Ok(SendOutcome::Sent), + Err(e) => { + if shutdown.is_cancelled() { + tracing::info!("pipeline stopped receiving during shutdown"); + Ok(SendOutcome::ShuttingDown) + } else { + Err(CdcError::Mongodb(format!("failed to send event: {e}"))) + } + } + } +} + +/// MongoDB CDC source implementing the Source trait. +/// +/// Orchestrates: check offset → snapshot if needed → stream change events. +/// Uses the official `mongodb` crate driver. Requires a replica set deployment. +pub struct MongodbSource { + config: MongodbSourceConfig, + offset_store: O, +} + +impl MongodbSource { + pub fn new(config: MongodbSourceConfig, offset_store: O) -> Self { + Self { + config, + offset_store, + } + } +} + +impl Source for MongodbSource { + async fn start( + &mut self, + sender: mpsc::Sender, + shutdown: CancellationToken, + ) -> Result<()> { + let client = mongodb::Client::with_uri_str(&self.config.connection_string) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to connect: {e}")))?; + + validate_collections(&client, &self.config.database, &self.config.collections).await?; + + // Phase 1: Determine start position (snapshot or resume) + let saved_offset = self.offset_store.load().await?; + + let stored = match saved_offset { + Some(offset_str) => Some(MongoOffset::from_json(&offset_str)?), + None => None, + }; + + let resume_point = match stored { + Some(MongoOffset::Token(token)) => { + tracing::info!("resuming change stream from saved resume token"); + ResumePoint::Token(token) + } + Some(MongoOffset::SnapshotComplete { cluster_time }) => { + tracing::info!( + cluster_time = %format!("{}:{}", cluster_time.time, cluster_time.increment), + "snapshot already complete, starting change stream at recorded cluster time" + ); + ResumePoint::AfterSnapshot { + snapshot_time: cluster_time.into(), + } + } + other => { + match &other { + Some(MongoOffset::SnapshotInProgress { collection, count }) => { + tracing::warn!( + %collection, + count, + "previous snapshot was interrupted; snapshots are not resumable, re-running it" + ); + } + _ => tracing::info!("no saved offset, performing initial snapshot"), + } + + let result = perform_snapshot( + &client, + &self.config.database, + &self.config.collections, + &sender, + &shutdown, + ) + .await?; + + if !result.completed { + return Ok(()); + } + + ResumePoint::AfterSnapshot { + snapshot_time: result.snapshot_time, + } + } + }; + + if shutdown.is_cancelled() { + return Ok(()); + } + + // Phase 2: Stream change events + tracing::info!( + database = %self.config.database, + "starting change stream" + ); + + let mut stream = match open_change_stream( + &client, + &self.config.database, + &self.config.collections, + &resume_point, + ) + .await + { + Ok(stream) => stream, + // Position gone from the oplog: re-snapshot, or every restart wedges. + Err(e) if is_resume_point_lost(&e) => { + tracing::warn!( + error = %e, + "stored change stream position is no longer in the oplog; re-running the snapshot" + ); + + let result = perform_snapshot( + &client, + &self.config.database, + &self.config.collections, + &sender, + &shutdown, + ) + .await?; + + if !result.completed { + return Ok(()); + } + + open_change_stream( + &client, + &self.config.database, + &self.config.collections, + &ResumePoint::AfterSnapshot { + snapshot_time: result.snapshot_time, + }, + ) + .await? + } + Err(e) => return Err(e), + }; + + pump_change_stream(&mut stream, &self.config.database, &sender, &shutdown).await + } +} diff --git a/src/source/mongodb/offset.rs b/src/source/mongodb/offset.rs new file mode 100644 index 0000000..e6a8621 --- /dev/null +++ b/src/source/mongodb/offset.rs @@ -0,0 +1,152 @@ +use mongodb::bson::Timestamp; +use mongodb::change_stream::event::ResumeToken; +use serde::{Deserialize, Serialize}; + +use crate::error::{CdcError, Result}; + +/// Cluster time split into its two components — `bson::Timestamp`'s own serde +/// representation depends on the serializer, so it does not round-trip as JSON. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClusterTime { + pub time: u32, + pub increment: u32, +} + +impl From for ClusterTime { + fn from(ts: Timestamp) -> Self { + Self { + time: ts.time, + increment: ts.increment, + } + } +} + +impl From for Timestamp { + fn from(ct: ClusterTime) -> Self { + Timestamp { + time: ct.time, + increment: ct.increment, + } + } +} + +/// The persisted offset for a MongoDB source. +/// +/// Tagged because `ResumeToken` deserializes from *any* JSON object: an +/// untagged snapshot offset would be read back as a bogus resume token. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MongoOffset { + /// Snapshots are not resumable mid-way — this re-runs the whole snapshot. + SnapshotInProgress { collection: String, count: u64 }, + /// All collections scanned; stream starts at the recorded cluster time. + SnapshotComplete { cluster_time: ClusterTime }, + /// Streaming position: an opaque change stream resume token. + Token(ResumeToken), +} + +impl MongoOffset { + pub fn to_json(&self) -> Result { + serde_json::to_string(self) + .map_err(|e| CdcError::Mongodb(format!("failed to serialize offset: {e}"))) + } + + pub fn from_json(s: &str) -> Result { + serde_json::from_str(s).map_err(|e| { + CdcError::Mongodb(format!( + "failed to parse stored offset ({e}); offset value: {s}" + )) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_token() -> ResumeToken { + // Resume tokens are opaque documents of the form {"_data": ""}. + serde_json::from_str(r#"{"_data":"8265F1A2B3000000012B"}"#).unwrap() + } + + #[test] + fn test_token_offset_round_trip() { + let offset = MongoOffset::Token(sample_token()); + let json = offset.to_json().unwrap(); + + match MongoOffset::from_json(&json).unwrap() { + MongoOffset::Token(token) => { + assert_eq!( + serde_json::to_string(&token).unwrap(), + serde_json::to_string(&sample_token()).unwrap() + ); + } + other => panic!("expected Token, got {other:?}"), + } + } + + #[test] + fn test_snapshot_complete_round_trip() { + let offset = MongoOffset::SnapshotComplete { + cluster_time: ClusterTime { + time: 1700000000, + increment: 7, + }, + }; + let json = offset.to_json().unwrap(); + + match MongoOffset::from_json(&json).unwrap() { + MongoOffset::SnapshotComplete { cluster_time } => { + assert_eq!(cluster_time.time, 1700000000); + assert_eq!(cluster_time.increment, 7); + } + other => panic!("expected SnapshotComplete, got {other:?}"), + } + } + + #[test] + fn test_snapshot_in_progress_round_trip() { + let offset = MongoOffset::SnapshotInProgress { + collection: "users".into(), + count: 1000, + }; + let json = offset.to_json().unwrap(); + + match MongoOffset::from_json(&json).unwrap() { + MongoOffset::SnapshotInProgress { collection, count } => { + assert_eq!(collection, "users"); + assert_eq!(count, 1000); + } + other => panic!("expected SnapshotInProgress, got {other:?}"), + } + } + + #[test] + fn test_snapshot_offset_is_not_parsed_as_resume_token() { + // The old (buggy) snapshot offset format must be rejected rather than + // silently accepted as a resume token. + let legacy = r#"{"snapshot_complete":"users","count":42}"#; + assert!(MongoOffset::from_json(legacy).is_err()); + } + + #[test] + fn test_arbitrary_json_is_rejected() { + for input in ["", "null", "\"token\"", "{}", r#"{"unknown":{}}"#] { + assert!( + MongoOffset::from_json(input).is_err(), + "expected rejection for {input}" + ); + } + } + + #[test] + fn test_cluster_time_timestamp_conversion() { + let ts = Timestamp { + time: 42, + increment: 3, + }; + let round_tripped: Timestamp = ClusterTime::from(ts).into(); + assert_eq!(round_tripped.time, 42); + assert_eq!(round_tripped.increment, 3); + } +} diff --git a/src/source/mongodb/snapshot.rs b/src/source/mongodb/snapshot.rs new file mode 100644 index 0000000..6b87b8d --- /dev/null +++ b/src/source/mongodb/snapshot.rs @@ -0,0 +1,156 @@ +use futures_util::StreamExt; +use mongodb::bson::{doc, Timestamp}; +use mongodb::Client; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use crate::error::{CdcError, Result}; +use crate::source::SourceEvent; + +use super::event_converter::snapshot_doc_to_cdc_event; +use super::offset::MongoOffset; +use super::{send_event, SendOutcome}; + +const SNAPSHOT_CHECKPOINT_INTERVAL: u64 = 1000; + +pub struct SnapshotResult { + /// Cluster time recorded before the scan; where the change stream starts. + pub snapshot_time: Timestamp, + /// False if the snapshot was interrupted by a shutdown signal. + pub completed: bool, +} + +/// Perform initial snapshot of the specified collections. +/// +/// Records the cluster time via `ping`, then scans each collection with +/// `find({})`. Progress checkpoints are `snapshot_in_progress`; the final one +/// is `snapshot_complete` and carries the cluster time to stream from. +pub async fn perform_snapshot( + client: &Client, + database: &str, + collections: &[String], + sender: &mpsc::Sender, + shutdown: &CancellationToken, +) -> Result { + let db = client.database(database); + + let ping_result = db + .run_command(doc! { "ping": 1 }) + .await + .map_err(|e| CdcError::Mongodb(format!("failed to ping for cluster time: {e}")))?; + + let snapshot_time = ping_result + .get("operationTime") + .and_then(|v| v.as_timestamp()) + .ok_or_else(|| { + CdcError::Mongodb("ping response missing operationTime — is this a replica set?".into()) + })?; + + tracing::info!( + snapshot_time = %format!("{}:{}", snapshot_time.time, snapshot_time.increment), + "recorded cluster time for snapshot" + ); + + let collection_names = if collections.is_empty() { + db.list_collection_names() + .await + .map_err(|e| CdcError::Mongodb(format!("failed to list collections: {e}")))? + } else { + collections.to_vec() + }; + + for coll_name in &collection_names { + if shutdown.is_cancelled() { + tracing::info!("shutdown received, aborting snapshot"); + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } + + tracing::info!(collection = %coll_name, "starting snapshot"); + let coll = db.collection::(coll_name); + + let mut cursor = coll + .find(doc! {}) + .await + .map_err(|e| CdcError::Mongodb(format!("snapshot find on {coll_name}: {e}")))?; + + let mut count: u64 = 0; + loop { + let next = tokio::select! { + _ = shutdown.cancelled() => { + tracing::info!(collection = %coll_name, count, "shutdown received, aborting snapshot"); + return Ok(SnapshotResult { snapshot_time, completed: false }); + } + next = cursor.next() => next, + }; + + let Some(result) = next else { + break; + }; + + let doc = result + .map_err(|e| CdcError::Mongodb(format!("snapshot cursor on {coll_name}: {e}")))?; + + let event = snapshot_doc_to_cdc_event(&doc, database, coll_name); + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Change(event), shutdown).await? + { + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } + + count += 1; + + if count.is_multiple_of(SNAPSHOT_CHECKPOINT_INTERVAL) { + let offset = MongoOffset::SnapshotInProgress { + collection: coll_name.clone(), + count, + } + .to_json()?; + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Checkpoint { offset }, shutdown).await? + { + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } + } + } + + // Still in progress until every collection has been scanned. + let offset = MongoOffset::SnapshotInProgress { + collection: coll_name.clone(), + count, + } + .to_json()?; + if let SendOutcome::ShuttingDown = + send_event(sender, SourceEvent::Checkpoint { offset }, shutdown).await? + { + return Ok(SnapshotResult { + snapshot_time, + completed: false, + }); + } + + tracing::info!(collection = %coll_name, count, "snapshot complete"); + } + + let offset = MongoOffset::SnapshotComplete { + cluster_time: snapshot_time.into(), + } + .to_json()?; + let completed = matches!( + send_event(sender, SourceEvent::Checkpoint { offset }, shutdown).await?, + SendOutcome::Sent + ); + + Ok(SnapshotResult { + snapshot_time, + completed, + }) +} diff --git a/tests/iceberg_integration.rs b/tests/iceberg_integration.rs index 217a320..e690a41 100644 --- a/tests/iceberg_integration.rs +++ b/tests/iceberg_integration.rs @@ -651,3 +651,88 @@ async fn test_iceberg_replication_multi_batch() { "expected at least 1 delete manifest, got {delete_count}" ); } + +// --------------------------------------------------------------------------- +// MongoDB Source → Iceberg Sink Tests +// --------------------------------------------------------------------------- + +/// Test that MongoDB-shaped CDC events (schemaless, _id PK, mixed types) write +/// to Iceberg correctly in CDC mode. Uses a MongoDB SourceConnectionConfig so +/// the type mapping goes through the canonical Mongodb dialect path. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_iceberg_cdc_mongodb_source() { + // We still need a Postgres for source schema inference when the sink + // auto-creates the table. Create a source table that mirrors what a + // MongoDB collection's inferred schema would look like. + let env = TestEnv::start(&[ + "CREATE TABLE public.mongo_users (_id TEXT, name TEXT, age INTEGER, active BOOLEAN)", + ]) + .await; + + let config = env.sink_config(); + let source_conn = env.source_connection(); + let mut sink = IcebergSink::new(config, SinkMode::Cdc, source_conn) + .await + .unwrap(); + + let table_id = make_table_id("public", "mongo_users"); + + // MongoDB-shaped events: _id as text, mixed types + let events = vec![ + CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: table_id.clone(), + op: ChangeOp::Snapshot, + new: Some(BTreeMap::from([ + ( + "_id".into(), + ColumnValue::Text("507f1f77bcf86cd799439011".into()), + ), + ("name".into(), ColumnValue::Text("Alice".into())), + ("age".into(), ColumnValue::Text("30".into())), + ("active".into(), ColumnValue::Text("true".into())), + ])), + old: None, + primary_key_columns: vec!["_id".into()], + }, + CdcEvent { + lsn: Lsn(200), + timestamp_us: 1_700_000_000_000_001, + xid: 0, + table: table_id.clone(), + op: ChangeOp::Insert, + new: Some(BTreeMap::from([ + ( + "_id".into(), + ColumnValue::Text("507f1f77bcf86cd799439012".into()), + ), + ("name".into(), ColumnValue::Text("Bob".into())), + ("age".into(), ColumnValue::Text("25".into())), + ("active".into(), ColumnValue::Text("false".into())), + ])), + old: None, + primary_key_columns: vec!["_id".into()], + }, + ]; + + sink.write_batch(&events).await.unwrap(); + sink.flush().await.unwrap(); + + // Verify table was created and has data + let catalog = build_test_catalog(&env).await; + let ns = NamespaceIdent::from_strs(["default"]).unwrap(); + let ident = TableIdent::new(ns, "cdc_test_mongo_users".to_string()); + let table = catalog.load_table(&ident).await.unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = snapshot + .load_manifest_list(table.file_io(), &table.metadata_ref()) + .await + .unwrap(); + assert!( + !manifest_list.entries().is_empty(), + "should have at least one manifest" + ); +} diff --git a/tests/kafka_integration.rs b/tests/kafka_integration.rs index 6746c99..287230b 100644 --- a/tests/kafka_integration.rs +++ b/tests/kafka_integration.rs @@ -42,6 +42,29 @@ async fn start_kafka() -> (ContainerAsync, String) { (container, bootstrap_servers) } +/// Create a MongoDB-shaped CDC event (schema=database, _id as PK, mixed types). +fn make_mongo_event(database: &str, collection: &str, id: &str) -> CdcEvent { + CdcEvent { + lsn: Lsn::ZERO, + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: database.into(), + name: collection.into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some(BTreeMap::from([ + ("_id".into(), ColumnValue::Text(id.into())), + ("name".into(), ColumnValue::Text("Alice".into())), + ("age".into(), ColumnValue::Int(30)), + ("active".into(), ColumnValue::Bool(true)), + ])), + old: None, + primary_key_columns: vec!["_id".into()], + } +} + fn make_test_event(schema: &str, table_name: &str, id: &str) -> CdcEvent { CdcEvent { lsn: Lsn::ZERO, @@ -168,3 +191,79 @@ async fn test_kafka_sink_message_key() { let key = msg.key_view::().unwrap().unwrap(); assert_eq!(key, "myschema.mytable"); } + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_kafka_sink_with_mongodb_events() { + let (_container, brokers) = start_kafka().await; + + let config = KafkaSinkConfig { + brokers: brokers.clone(), + topic_prefix: "mongo".into(), + properties: Default::default(), + }; + let mut sink = KafkaSink::new(config).unwrap(); + + // Write MongoDB-shaped events from two collections + let events = vec![ + make_mongo_event("demo", "users", "507f1f77bcf86cd799439011"), + make_mongo_event("demo", "users", "507f1f77bcf86cd799439012"), + make_mongo_event("demo", "orders", "607f1f77bcf86cd799439099"), + ]; + sink.write_batch(&events).await.unwrap(); + sink.flush().await.unwrap(); + + // Consume and verify topic routing: {prefix}.{database}.{collection} + let consumer: StreamConsumer = ClientConfig::new() + .set("bootstrap.servers", &brokers) + .set("group.id", "test-mongo-topics") + .set("auto.offset.reset", "earliest") + .create() + .unwrap(); + + consumer + .subscribe(&["mongo.demo.users", "mongo.demo.orders"]) + .unwrap(); + + let mut users_messages = Vec::new(); + let mut orders_messages = Vec::new(); + + let consume_result = timeout(Duration::from_secs(15), async { + loop { + if let Ok(msg) = tokio::time::timeout(Duration::from_secs(2), consumer.recv()).await { + let msg = msg.unwrap(); + let topic = msg.topic().to_string(); + let payload = msg.payload_view::().unwrap().unwrap().to_string(); + if topic == "mongo.demo.users" { + users_messages.push(payload); + } else if topic == "mongo.demo.orders" { + orders_messages.push(payload); + } + if users_messages.len() >= 2 && !orders_messages.is_empty() { + break; + } + } + } + }) + .await; + + assert!(consume_result.is_ok(), "failed to consume all messages"); + assert_eq!(users_messages.len(), 2); + assert_eq!(orders_messages.len(), 1); + + // Verify JSON structure contains MongoDB fields + // Kafka uses nested format: {metadata, new: {values, types}, old} + let user_json: serde_json::Value = serde_json::from_str(&users_messages[0]).unwrap(); + assert_eq!(user_json["metadata"]["table"], "users"); + assert_eq!(user_json["metadata"]["schema"], "demo"); + assert_eq!(user_json["metadata"]["op"], "I"); + // MongoDB _id should appear in new.values + assert!(user_json["new"]["values"]["_id"].is_string()); + assert_eq!(user_json["new"]["values"]["name"], "Alice"); + assert_eq!(user_json["new"]["values"]["age"], 30); + assert_eq!(user_json["new"]["values"]["active"], true); + // Types should reflect ColumnValue variants + assert_eq!(user_json["new"]["types"]["_id"], "Text"); + assert_eq!(user_json["new"]["types"]["age"], "Int"); + assert_eq!(user_json["new"]["types"]["active"], "Bool"); +} diff --git a/tests/mongodb_integration.rs b/tests/mongodb_integration.rs new file mode 100644 index 0000000..20a3ede --- /dev/null +++ b/tests/mongodb_integration.rs @@ -0,0 +1,1344 @@ +//! Integration tests for the full CDC pipeline with MongoDB. +//! +//! These tests require Docker to be running. Run with: +//! cargo test -- --ignored +//! +//! Each test spins up a MongoDB container configured as a single-node replica set. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use mongodb::bson::doc; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, ImageExt}; +use testcontainers_modules::mongo::Mongo; +use tokio::time::timeout; +use tokio_util::sync::CancellationToken; + +use cdcflow::config::{MongodbSourceConfig, PostgresSinkConfig, SinkMode, SourceConnectionConfig}; +use cdcflow::error::Result; +use cdcflow::event::{CdcEvent, ChangeOp, ColumnValue}; +use cdcflow::offset::memory::MemoryOffsetStore; +use cdcflow::offset::OffsetStore; +use cdcflow::pipeline::Pipeline; +use cdcflow::sink::postgres::PostgresSink; +use cdcflow::sink::Sink; +use cdcflow::source::mongodb::offset::{ClusterTime, MongoOffset}; +use cdcflow::source::mongodb::MongodbSource; + +/// A sink that collects events into a shared Vec for assertions. +struct CollectorSink { + events: Arc>>, +} + +impl CollectorSink { + fn new() -> (Self, Arc>>) { + let events = Arc::new(Mutex::new(Vec::new())); + ( + Self { + events: events.clone(), + }, + events, + ) + } +} + +impl Sink for CollectorSink { + async fn write_batch(&mut self, events: &[CdcEvent]) -> Result<()> { + for event in events { + let json = serde_json::to_string_pretty(event).unwrap(); + println!("\n--- CDC Event ---\n{json}"); + } + self.events.lock().unwrap().extend_from_slice(events); + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + Ok(()) + } +} + +/// Start a MongoDB container configured as a single-node replica set. +async fn start_mongodb() -> (ContainerAsync, String, u16) { + let container = Mongo::default() + .with_tag("8") + .with_cmd(vec![ + "mongod".to_string(), + "--replSet".to_string(), + "rs0".to_string(), + "--bind_ip_all".to_string(), + ]) + .start() + .await + .expect("failed to start mongodb container"); + + let host_port = container + .get_host_port_ipv4(27017) + .await + .expect("failed to get host port"); + + let host = container + .get_host() + .await + .expect("failed to get host") + .to_string(); + + // Initialize replica set — use 127.0.0.1:{mapped_port} as the member host + // so MongoDB recognizes itself. The container binds to 0.0.0.0 and the + // mapped port is forwarded from the host, making this work with directConnection. + let conn_str = format!("mongodb://{}:{}/?directConnection=true", host, host_port); + let client = mongodb::Client::with_uri_str(&conn_str) + .await + .expect("failed to connect to mongodb"); + + let admin_db = client.database("admin"); + + // Use replSetInitiate with "localhost:27017" — the address MongoDB sees + // internally. We connect from the host via the mapped port using + // directConnection=true, but the replica set member must use the address + // visible inside the container. + admin_db + .run_command(doc! { + "replSetInitiate": { + "_id": "rs0", + "members": [{ "_id": 0, "host": "localhost:27017" }] + } + }) + .await + .expect("failed to initiate replica set"); + + // Wait for primary election + for _ in 0..30 { + tokio::time::sleep(Duration::from_secs(1)).await; + if let Ok(status) = admin_db.run_command(doc! { "replSetGetStatus": 1 }).await { + if let Ok(members) = status.get_array("members") { + for member in members { + if let Some(doc) = member.as_document() { + if doc.get_str("stateStr").ok() == Some("PRIMARY") { + // Primary elected, ready to go + return (container, host.to_string(), host_port); + } + } + } + } + } + } + + panic!("MongoDB replica set did not elect a primary within 30 seconds"); +} + +/// Build a connection string for the test container. +fn connection_string(host: &str, port: u16) -> String { + format!( + "mongodb://{}:{}/?replicaSet=rs0&directConnection=true", + host, port + ) +} + +/// Get a mongodb::Client connected to the test container. +async fn get_client(host: &str, port: u16) -> mongodb::Client { + let conn = connection_string(host, port); + mongodb::Client::with_uri_str(&conn) + .await + .expect("failed to create mongodb client") +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_snapshot_and_stream() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + // Insert initial documents + let db = client.database("testdb"); + let coll = db.collection::("test_coll"); + + // Enable pre/post images + db.run_command(doc! { + "collMod": "test_coll", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .ok(); // May fail if collection doesn't exist yet, that's fine + + // Drop and recreate with pre/post images enabled + coll.drop().await.ok(); + db.create_collection("test_coll") + .await + .expect("failed to create collection"); + db.run_command(doc! { + "collMod": "test_coll", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .expect("failed to enable change stream images"); + + coll.insert_many(vec![ + doc! { "name": "Alice", "age": 30 }, + doc! { "name": "Bob", "age": 25 }, + ]) + .await + .expect("failed to insert test data"); + + // Set up pipeline + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "testdb".into(), + collections: vec!["test_coll".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait for snapshot to complete + let snapshot_received = timeout(Duration::from_secs(30), async { + loop { + { + let evts = events.lock().unwrap(); + let snapshot_count = evts.iter().filter(|e| e.op == ChangeOp::Snapshot).count(); + if snapshot_count >= 2 { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await; + assert!( + snapshot_received.is_ok(), + "timed out waiting for snapshot events" + ); + + // Verify snapshot events + { + let evts = events.lock().unwrap(); + let snapshots: Vec<&CdcEvent> = + evts.iter().filter(|e| e.op == ChangeOp::Snapshot).collect(); + assert_eq!(snapshots.len(), 2, "expected 2 snapshot events"); + for snap in &snapshots { + assert_eq!(snap.table.schema, "testdb"); + assert_eq!(snap.table.name, "test_coll"); + assert_eq!(snap.primary_key_columns, vec!["_id"]); + assert!(snap.new.is_some()); + } + } + + // Now insert a new document to test streaming + coll.insert_one(doc! { "name": "Charlie", "age": 35 }) + .await + .expect("failed to insert streaming doc"); + + // Wait for the "Charlie" insert event specifically + let insert_received = timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + let has_charlie = evts.iter().any(|e| { + e.op == ChangeOp::Insert + && e.new + .as_ref() + .and_then(|r| r.get("name")) + .map(|v| matches!(v, ColumnValue::Text(s) if s == "Charlie")) + .unwrap_or(false) + }); + if has_charlie { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await; + assert!( + insert_received.is_ok(), + "timed out waiting for Charlie insert event" + ); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_update_and_delete() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("testdb2"); + + // Create collection with pre/post images + db.create_collection("items") + .await + .expect("failed to create collection"); + db.run_command(doc! { + "collMod": "items", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .expect("failed to enable change stream images"); + + let coll = db.collection::("items"); + + // Insert initial doc + coll.insert_one(doc! { "_id": "item1", "name": "Widget", "price": 10.0 }) + .await + .expect("failed to insert"); + + // Set up pipeline + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "testdb2".into(), + collections: vec!["items".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait for snapshot + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Snapshot) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for snapshot"); + + // Update the document + coll.update_one(doc! { "_id": "item1" }, doc! { "$set": { "price": 15.0 } }) + .await + .expect("failed to update"); + + // Wait for update event + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Update) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for update event"); + + // Verify update event + { + let evts = events.lock().unwrap(); + let updates: Vec<&CdcEvent> = evts.iter().filter(|e| e.op == ChangeOp::Update).collect(); + assert!(!updates.is_empty(), "expected update event"); + let update = updates.last().unwrap(); + // With updateLookup, full_document should have the updated doc + let new_row = update.new.as_ref().expect("update should have new row"); + assert_eq!( + new_row.get("name"), + Some(&ColumnValue::Text("Widget".into())) + ); + } + + // Delete the document + coll.delete_one(doc! { "_id": "item1" }) + .await + .expect("failed to delete"); + + // Wait for delete event + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Delete) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for delete event"); + + // Verify delete event + { + let evts = events.lock().unwrap(); + let deletes: Vec<&CdcEvent> = evts.iter().filter(|e| e.op == ChangeOp::Delete).collect(); + assert!(!deletes.is_empty(), "expected delete event"); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_resume_from_offset() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("testdb3"); + db.create_collection("resume_coll") + .await + .expect("failed to create collection"); + db.run_command(doc! { + "collMod": "resume_coll", + "changeStreamPreAndPostImages": { "enabled": true } + }) + .await + .expect("failed to enable change stream images"); + + let coll = db.collection::("resume_coll"); + + // Insert initial doc + coll.insert_one(doc! { "name": "First" }) + .await + .expect("failed to insert"); + + let conn_str = connection_string(&host, port); + + // Run 1: Snapshot + stream, then stop + let offset_store = MemoryOffsetStore::new(); + { + let config = MongodbSourceConfig { + connection_string: conn_str.clone(), + database: "testdb3".into(), + collections: vec!["resume_coll".into()], + }; + + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store.clone()); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait for snapshot + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Snapshot) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for snapshot"); + + // Insert while streaming to generate an offset + coll.insert_one(doc! { "name": "During" }) + .await + .expect("failed to insert"); + + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + if evts.iter().any(|e| e.op == ChangeOp::Insert) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for insert"); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; + } + + // Verify offset was saved + let saved_offset = offset_store.load().await.expect("failed to load offset"); + assert!(saved_offset.is_some(), "offset should have been saved"); + + // Insert a doc while pipeline is down + coll.insert_one(doc! { "name": "WhileDown" }) + .await + .expect("failed to insert while down"); + + // Run 2: Resume from offset + { + let config = MongodbSourceConfig { + connection_string: conn_str, + database: "testdb3".into(), + collections: vec!["resume_coll".into()], + }; + + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Should receive the "WhileDown" insert without re-snapshotting + timeout(Duration::from_secs(15), async { + loop { + { + let evts = events.lock().unwrap(); + let insert_names: Vec = evts + .iter() + .filter(|e| e.op == ChangeOp::Insert) + .filter_map(|e| { + e.new + .as_ref() + .and_then(|r| r.get("name")) + .and_then(|v| match v { + ColumnValue::Text(s) => Some(s.clone()), + _ => None, + }) + }) + .collect(); + if insert_names.contains(&"WhileDown".to_string()) { + return; + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("timed out waiting for WhileDown insert after resume"); + + // Verify no snapshot events (we resumed, not re-snapshotted) + { + let evts = events.lock().unwrap(); + let snapshot_count = evts.iter().filter(|e| e.op == ChangeOp::Snapshot).count(); + assert_eq!( + snapshot_count, 0, + "should not have snapshot events after resume" + ); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; + } +} + +/// Poll `check` until it returns true or the timeout expires. +async fn wait_until(secs: u64, label: &str, mut check: F) +where + F: FnMut() -> bool, +{ + let result = timeout(Duration::from_secs(secs), async { + loop { + if check() { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await; + assert!(result.is_ok(), "timed out waiting for {label}"); +} + +/// Without pre-images (the MongoDB default), update and delete must still +/// carry `_id` in `old` so sinks can target the row. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_update_and_delete_without_pre_images() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("no_preimage_db"); + // Deliberately no collMod: pre-images stay disabled. + db.create_collection("items") + .await + .expect("failed to create collection"); + let coll = db.collection::("items"); + coll.insert_one(doc! { "_id": "item1", "name": "Widget", "price": 10.0 }) + .await + .expect("failed to insert"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "no_preimage_db".into(), + collections: vec!["items".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "snapshot", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Snapshot) + }) + .await; + } + + coll.update_one(doc! { "_id": "item1" }, doc! { "$set": { "price": 15.0 } }) + .await + .expect("failed to update"); + + { + let events = events.clone(); + wait_until(20, "update event", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Update) + }) + .await; + } + + { + let evts = events.lock().unwrap(); + let update = evts.iter().find(|e| e.op == ChangeOp::Update).unwrap(); + let old = update + .old + .as_ref() + .expect("update must carry an old row built from documentKey"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("item1".into()))); + } + + coll.delete_one(doc! { "_id": "item1" }) + .await + .expect("failed to delete"); + + { + let events = events.clone(); + wait_until(20, "delete event", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Delete) + }) + .await; + } + + { + let evts = events.lock().unwrap(); + let delete = evts.iter().find(|e| e.op == ChangeOp::Delete).unwrap(); + let old = delete + .old + .as_ref() + .expect("delete must carry an old row built from documentKey"); + assert_eq!(old.get("_id"), Some(&ColumnValue::Text("item1".into()))); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// Current cluster time — what a completed snapshot records in its offset. +async fn current_cluster_time(client: &mongodb::Client, database: &str) -> ClusterTime { + let ping = client + .database(database) + .run_command(doc! { "ping": 1 }) + .await + .expect("ping failed"); + let ts = ping + .get("operationTime") + .and_then(|v| v.as_timestamp()) + .expect("ping response missing operationTime"); + ClusterTime::from(ts) +} + +/// A `snapshot_complete` offset resumes from its cluster time, no re-snapshot. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_resumes_from_snapshot_complete_offset() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("snapshot_only_db"); + db.create_collection("docs") + .await + .expect("failed to create collection"); + let coll = db.collection::("docs"); + coll.insert_one(doc! { "name": "BeforeOffset" }) + .await + .expect("failed to insert"); + + // Exactly what the source persists once a snapshot finishes with no change + // events behind it. + let cluster_time = current_cluster_time(&client, "snapshot_only_db").await; + let offset_store = MemoryOffsetStore::with_offset( + MongoOffset::SnapshotComplete { cluster_time } + .to_json() + .unwrap(), + ); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "snapshot_only_db".into(), + collections: vec!["docs".into()], + }; + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Written after the recorded cluster time, so it must arrive on the stream. + tokio::time::sleep(Duration::from_secs(2)).await; + coll.insert_one(doc! { "name": "AfterOffset" }) + .await + .expect("failed to insert"); + + { + let events = events.clone(); + wait_until(30, "AfterOffset insert", move || { + events.lock().unwrap().iter().any(|e| { + e.op == ChangeOp::Insert + && e.new + .as_ref() + .and_then(|r| r.get("name")) + .map(|v| matches!(v, ColumnValue::Text(s) if s == "AfterOffset")) + .unwrap_or(false) + }) + }) + .await; + } + + { + let evts = events.lock().unwrap(); + assert_eq!( + evts.iter().filter(|e| e.op == ChangeOp::Snapshot).count(), + 0, + "a completed snapshot must not be re-run on restart" + ); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// Snapshots are not resumable mid-way, so `snapshot_in_progress` re-runs it. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_reruns_interrupted_snapshot() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("interrupted_db"); + db.create_collection("docs") + .await + .expect("failed to create collection"); + db.collection::("docs") + .insert_many(vec![doc! { "name": "A" }, doc! { "name": "B" }]) + .await + .expect("failed to insert"); + + let offset_store = MemoryOffsetStore::with_offset( + MongoOffset::SnapshotInProgress { + collection: "docs".into(), + count: 1, + } + .to_json() + .unwrap(), + ); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "interrupted_db".into(), + collections: vec!["docs".into()], + }; + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "re-run snapshot of both documents", move || { + events + .lock() + .unwrap() + .iter() + .filter(|e| e.op == ChangeOp::Snapshot) + .count() + >= 2 + }) + .await; + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// An uninterpretable offset must fail loudly, not become a bogus token. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_rejects_unrecognized_offset() { + let (_container, host, port) = start_mongodb().await; + + let offset_store = + MemoryOffsetStore::with_offset(r#"{"snapshot_complete":"docs","count":42}"#.to_string()); + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "whatever_db".into(), + collections: vec![], + }; + let (sink, _events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let result = timeout(Duration::from_secs(30), pipeline.run(shutdown)) + .await + .expect("pipeline should fail fast on an unreadable offset"); + + let err = result.expect_err("an unreadable offset must be an error"); + assert!( + err.to_string().contains("failed to parse stored offset"), + "unexpected error: {err}" + ); +} + +/// An empty `collections` list snapshots and streams every collection in the database. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_all_collections_when_filter_empty() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("all_colls_db"); + for name in ["alpha", "beta"] { + db.create_collection(name) + .await + .expect("failed to create collection"); + db.collection::(name) + .insert_one(doc! { "name": name }) + .await + .expect("failed to insert"); + } + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "all_colls_db".into(), + collections: vec![], // all collections + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "snapshot of both collections", move || { + let evts = events.lock().unwrap(); + let tables: std::collections::HashSet = evts + .iter() + .filter(|e| e.op == ChangeOp::Snapshot) + .map(|e| e.table.name.clone()) + .collect(); + tables.contains("alpha") && tables.contains("beta") + }) + .await; + } + + // A change in either collection must stream through too. + db.collection::("beta") + .insert_one(doc! { "name": "beta2" }) + .await + .expect("failed to insert"); + + { + let events = events.clone(); + wait_until(20, "streamed insert on beta", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Insert && e.table.name == "beta") + }) + .await; + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// A non-empty `collections` list must exclude changes from other collections. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_collection_filter_excludes_others() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("filter_db"); + for name in ["watched", "ignored"] { + db.create_collection(name) + .await + .expect("failed to create collection"); + } + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "filter_db".into(), + collections: vec!["watched".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Give the change stream time to open before writing. + tokio::time::sleep(Duration::from_secs(3)).await; + + db.collection::("ignored") + .insert_one(doc! { "name": "nope" }) + .await + .expect("failed to insert"); + db.collection::("watched") + .insert_one(doc! { "name": "yes" }) + .await + .expect("failed to insert"); + + { + let events = events.clone(); + wait_until(20, "insert on watched collection", move || { + events + .lock() + .unwrap() + .iter() + .any(|e| e.op == ChangeOp::Insert && e.table.name == "watched") + }) + .await; + } + + { + let evts = events.lock().unwrap(); + assert!( + evts.iter().all(|e| e.table.name != "ignored"), + "events from unwatched collections must be filtered out" + ); + } + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(5), handle).await; +} + +/// Shutdown during a snapshot must stop it promptly, mid-scan. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_shutdown_during_snapshot() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("big_snapshot_db"); + let coll = db.collection::("many"); + let docs: Vec = (0..20_000) + .map(|i| doc! { "i": i, "payload": "x".repeat(200) }) + .collect(); + coll.insert_many(docs).await.expect("failed to seed docs"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "big_snapshot_db".into(), + collections: vec!["many".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Wait until the snapshot is clearly underway, then cancel mid-scan. + { + let events = events.clone(); + wait_until(30, "snapshot to start", move || { + !events.lock().unwrap().is_empty() + }) + .await; + } + shutdown.cancel(); + + let result = timeout(Duration::from_secs(10), handle) + .await + .expect("pipeline did not stop within 10s of cancellation"); + result + .expect("pipeline task panicked") + .expect("pipeline returned an error"); + + let seen = events.lock().unwrap().len(); + assert!( + seen < 20_000, + "snapshot should have been interrupted, but emitted all {seen} documents" + ); +} + +/// End-to-end replication into PostgreSQL with pre-images disabled. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_to_postgres_replication() { + use testcontainers_modules::postgres::Postgres; + use tokio_postgres::NoTls; + + let (_mongo, mongo_host, mongo_port) = start_mongodb().await; + let pg_container = Postgres::default() + .with_tag("18-alpine") + .start() + .await + .expect("failed to start postgres container"); + let pg_port = pg_container + .get_host_port_ipv4(5432) + .await + .expect("failed to get pg port"); + let pg_host = pg_container + .get_host() + .await + .expect("failed to get pg host") + .to_string(); + + let mongo_client = get_client(&mongo_host, mongo_port).await; + let db = mongo_client.database("repl_db"); + db.create_collection("users") + .await + .expect("failed to create collection"); + let coll = db.collection::("users"); + coll.insert_many(vec![ + doc! { "_id": "u1", "name": "Alice", "age": 30 }, + doc! { "_id": "u2", "name": "Bob", "age": 25 }, + ]) + .await + .expect("failed to seed docs"); + + let mongo_conn = connection_string(&mongo_host, mongo_port); + let sink = PostgresSink::new( + PostgresSinkConfig { + host: pg_host.clone(), + port: pg_port, + user: "postgres".into(), + password: Some("postgres".into()), + database: "postgres".into(), + schema: "public".into(), + table_prefix: "".into(), + }, + SinkMode::Replication, + Some(SourceConnectionConfig::Mongodb { + url: mongo_conn.clone(), + }), + ) + .await + .expect("failed to create postgres sink"); + + let config = MongodbSourceConfig { + connection_string: mongo_conn, + database: "repl_db".into(), + collections: vec!["users".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Verification client. + let (pg, connection) = tokio_postgres::connect( + &format!( + "host={} port={} user=postgres password=postgres dbname=postgres", + pg_host, pg_port + ), + NoTls, + ) + .await + .expect("failed to connect to postgres"); + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("pg connection error: {e}"); + } + }); + + async fn count_users(pg: &tokio_postgres::Client) -> i64 { + pg.query_one("SELECT count(*)::bigint FROM public.users", &[]) + .await + .map(|row| row.get::<_, i64>(0)) + .unwrap_or(-1) + } + + // Snapshot replicated. + timeout(Duration::from_secs(60), async { + loop { + if count_users(&pg).await == 2 { + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .expect("timed out waiting for snapshot rows in postgres"); + + // Update (no pre-image available) must update the existing row, not add one. + coll.update_one(doc! { "_id": "u1" }, doc! { "$set": { "age": 31 } }) + .await + .expect("failed to update"); + + timeout(Duration::from_secs(60), async { + loop { + let row = pg + .query_opt("SELECT age FROM public.users WHERE _id = 'u1'", &[]) + .await + .expect("query failed"); + if let Some(row) = row { + let age: i64 = match row.try_get::<_, i64>(0) { + Ok(v) => v, + Err(_) => row.get::<_, i32>(0) as i64, + }; + if age == 31 && count_users(&pg).await == 2 { + return; + } + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .expect("timed out waiting for the update to replicate"); + + // Delete (no pre-image available) must remove the row by its `_id`. + coll.delete_one(doc! { "_id": "u2" }) + .await + .expect("failed to delete"); + + timeout(Duration::from_secs(60), async { + loop { + if count_users(&pg).await == 1 { + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .expect("timed out waiting for the delete to replicate"); + + let remaining: String = pg + .query_one("SELECT _id FROM public.users", &[]) + .await + .expect("query failed") + .get(0); + assert_eq!(remaining, "u1", "the wrong row was deleted"); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(10), handle).await; +} + +/// A missing collection must fail fast, not run forever replicating nothing. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_rejects_unknown_collection() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("typo_db"); + db.create_collection("users") + .await + .expect("failed to create collection"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "typo_db".into(), + collections: vec!["Users".into()], // wrong case + }; + + let (sink, _events) = CollectorSink::new(); + let offset_store = MemoryOffsetStore::new(); + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store); + + let result = timeout( + Duration::from_secs(30), + pipeline.run(CancellationToken::new()), + ) + .await + .expect("pipeline should fail fast on an unknown collection"); + + let err = result.expect_err("an unknown collection must be an error"); + assert!( + err.to_string().contains("not found in database typo_db"), + "unexpected error: {err}" + ); +} + +/// Writes to filtered-out collections must still advance the stored offset, +/// otherwise it ages out of the oplog while the watched collection is idle. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_offset_advances_while_watched_collection_idle() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("idle_db"); + for name in ["watched", "busy"] { + db.create_collection(name) + .await + .expect("failed to create collection"); + } + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "idle_db".into(), + collections: vec!["watched".into()], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, _events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store.clone()); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + // Let the snapshot finish and the stream open. + tokio::time::sleep(Duration::from_secs(3)).await; + let baseline = offset_store.load().await.unwrap(); + + // Traffic only on the collection that is filtered out. + for i in 0..5 { + db.collection::("busy") + .insert_one(doc! { "i": i }) + .await + .expect("failed to insert"); + tokio::time::sleep(Duration::from_millis(200)).await; + } + + timeout(Duration::from_secs(30), async { + loop { + let current = offset_store.load().await.unwrap(); + if current.is_some() && current != baseline { + return; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + }) + .await + .expect("timed out waiting for the offset to advance past the filtered writes"); + + let advanced = offset_store.load().await.unwrap().unwrap(); + assert!( + matches!(MongoOffset::from_json(&advanced), Ok(MongoOffset::Token(_))), + "expected a resume token offset, got {advanced}" + ); + + shutdown.cancel(); + let _ = timeout(Duration::from_secs(10), handle).await; +} + +/// Dropping the database ends the stream with an `invalidate` event, whose +/// token must never be stored: the server refuses it in `resumeAfter`. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mongodb_invalidate_token_is_not_persisted() { + let (_container, host, port) = start_mongodb().await; + let client = get_client(&host, port).await; + + let db = client.database("dropped_db"); + db.create_collection("docs") + .await + .expect("failed to create collection"); + let coll = db.collection::("docs"); + coll.insert_one(doc! { "name": "First" }) + .await + .expect("failed to insert"); + + let config = MongodbSourceConfig { + connection_string: connection_string(&host, port), + database: "dropped_db".into(), + collections: vec![], + }; + + let offset_store = MemoryOffsetStore::new(); + let (sink, events) = CollectorSink::new(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let source = MongodbSource::new(config, offset_store.clone()); + let pipeline = Pipeline::new(source, sink, offset_store.clone()); + let handle = tokio::spawn(async move { pipeline.run(shutdown_clone).await }); + + { + let events = events.clone(); + wait_until(30, "snapshot", move || !events.lock().unwrap().is_empty()).await; + } + tokio::time::sleep(Duration::from_secs(2)).await; + + db.drop().await.expect("failed to drop database"); + + let result = timeout(Duration::from_secs(30), handle) + .await + .expect("pipeline should stop after the stream is invalidated") + .expect("pipeline task panicked"); + assert!( + result.is_err(), + "a server-closed change stream must surface as an error" + ); + shutdown.cancel(); + + // Whatever was stored must still be usable as a resume point. + let saved = offset_store + .load() + .await + .unwrap() + .expect("an offset should have been stored"); + match MongoOffset::from_json(&saved).expect("stored offset must parse") { + MongoOffset::Token(token) => { + client + .database("dropped_db") + .watch() + .resume_after(token) + .await + .expect("stored resume token must be accepted by the server"); + } + MongoOffset::SnapshotComplete { .. } | MongoOffset::SnapshotInProgress { .. } => {} + } +} diff --git a/tests/postgres_sink_integration.rs b/tests/postgres_sink_integration.rs index bf62a10..97b8eae 100644 --- a/tests/postgres_sink_integration.rs +++ b/tests/postgres_sink_integration.rs @@ -163,6 +163,34 @@ fn make_truncate_event(schema: &str, table: &str) -> CdcEvent { } } +/// Create a MongoDB-shaped insert event (schema=database, _id as PK, mixed types). +fn make_mongo_insert(database: &str, collection: &str, id: &str, name: &str) -> CdcEvent { + CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: database.into(), + name: collection.into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text(id.into())), + ("name".to_string(), ColumnValue::Text(name.into())), + ("age".to_string(), ColumnValue::Int(30)), + ("active".to_string(), ColumnValue::Bool(true)), + ("score".to_string(), ColumnValue::float(95.5)), + ] + .into_iter() + .collect(), + ), + old: None, + primary_key_columns: vec!["_id".into()], + } +} + // ───────────────────────────────────────────────── // CDC Mode Tests // ───────────────────────────────────────────────── @@ -890,3 +918,345 @@ async fn test_replication_mode_schema_evolution() { let phone2: Option<&str> = rows[1].get("phone"); assert_eq!(phone2, Some("555-1234")); } + +// ───────────────────────────────────────────────── +// MongoDB Source → PostgreSQL Sink Tests +// ───────────────────────────────────────────────── + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_cdc_mode_mongodb_source_events() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + // MongoDB source has no real source DB for schema inference — we use + // SourceConnectionConfig::Mongodb which triggers canonical type mapping. + // But the PG sink in CDC mode needs a source_connection for auto-creating + // tables. Since we don't have a real MongoDB, pre-create the target table + // with the expected schema instead. + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"users\" ( + \"_cdc_op\" text, + \"_cdc_lsn\" bigint, + \"_cdc_timestamp_us\" bigint, + \"_cdc_snapshot\" boolean, + \"_cdc_schema\" text, + \"_cdc_table\" text, + \"_cdc_primary_key_columns\" text, + \"_id\" text, + \"name\" text, + \"age\" integer, + \"active\" boolean, + \"score\" double precision, + \"_old__id\" text, + \"_old_name\" text, + \"_old_age\" integer, + \"_old_active\" boolean, + \"_old_score\" double precision + )", + &[], + ) + .await + .unwrap(); + + let config = make_sink_config(&host, port); + let mut sink = PostgresSink::new(config, SinkMode::Cdc, None) + .await + .unwrap(); + + let events = vec![ + make_mongo_insert("demo", "users", "507f1f77bcf86cd799439011", "Alice"), + make_mongo_insert("demo", "users", "507f1f77bcf86cd799439012", "Bob"), + ]; + + sink.write_batch(&events).await.unwrap(); + sink.flush().await.unwrap(); + + // Verify rows landed with correct typed values + let rows = client + .query( + "SELECT \"_cdc_op\", \"_id\", \"name\", \"age\", \"active\", \"score\" \ + FROM \"public\".\"users\" ORDER BY \"_id\"", + &[], + ) + .await + .unwrap(); + + assert_eq!(rows.len(), 2); + + let op: &str = rows[0].get("_cdc_op"); + assert_eq!(op, "I"); + let id: Option<&str> = rows[0].get("_id"); + assert_eq!(id, Some("507f1f77bcf86cd799439011")); + let name: Option<&str> = rows[0].get("name"); + assert_eq!(name, Some("Alice")); + let age: Option = rows[0].get("age"); + assert_eq!(age, Some(30)); + let active: Option = rows[0].get("active"); + assert_eq!(active, Some(true)); + let score: Option = rows[0].get("score"); + assert_eq!(score, Some(95.5)); +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_replication_mode_mongodb_source_upsert_delete() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + // Pre-create table with MongoDB-style schema (_id as PK) + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"items\" ( + \"_id\" TEXT NOT NULL, + \"name\" TEXT, + \"price\" DOUBLE PRECISION, + PRIMARY KEY (\"_id\") + )", + &[], + ) + .await + .unwrap(); + + let config = make_sink_config(&host, port); + let mut sink = PostgresSink::new(config, SinkMode::Replication, None) + .await + .unwrap(); + + // Insert + let insert = CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text("item1".into())), + ("name".to_string(), ColumnValue::Text("Widget".into())), + ("price".to_string(), ColumnValue::float(19.99)), + ] + .into_iter() + .collect(), + ), + old: None, + primary_key_columns: vec!["_id".into()], + }; + sink.write_batch(&[insert]).await.unwrap(); + sink.flush().await.unwrap(); + + let count: i64 = client + .query_one("SELECT COUNT(*) FROM \"public\".\"items\"", &[]) + .await + .unwrap() + .get(0); + assert_eq!(count, 1); + + // Update (upsert) + let update = CdcEvent { + lsn: Lsn(200), + timestamp_us: 1_700_000_000_000_001, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Update, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text("item1".into())), + ("name".to_string(), ColumnValue::Text("Widget Pro".into())), + ("price".to_string(), ColumnValue::float(29.99)), + ] + .into_iter() + .collect(), + ), + old: Some( + vec![("_id".to_string(), ColumnValue::Text("item1".into()))] + .into_iter() + .collect(), + ), + primary_key_columns: vec!["_id".into()], + }; + sink.write_batch(&[update]).await.unwrap(); + sink.flush().await.unwrap(); + + // Still 1 row, but updated + let row = client + .query_one("SELECT \"name\", \"price\" FROM \"public\".\"items\"", &[]) + .await + .unwrap(); + let name: Option<&str> = row.get("name"); + assert_eq!(name, Some("Widget Pro")); + let price: Option = row.get("price"); + assert_eq!(price, Some(29.99)); + + // Delete + let delete = CdcEvent { + lsn: Lsn(300), + timestamp_us: 1_700_000_000_000_002, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Delete, + new: None, + old: Some( + vec![("_id".to_string(), ColumnValue::Text("item1".into()))] + .into_iter() + .collect(), + ), + primary_key_columns: vec!["_id".into()], + }; + sink.write_batch(&[delete]).await.unwrap(); + sink.flush().await.unwrap(); + + let count: i64 = client + .query_one("SELECT COUNT(*) FROM \"public\".\"items\"", &[]) + .await + .unwrap() + .get(0); + assert_eq!(count, 0); +} + +/// A field missing from a batch of MongoDB documents is not a dropped column — +/// the target column and its data must survive. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_replication_mode_mongodb_sparse_field_does_not_drop_column() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"items\" ( + \"_id\" TEXT NOT NULL, + \"name\" TEXT, + \"nickname\" TEXT, + PRIMARY KEY (\"_id\") + )", + &[], + ) + .await + .unwrap(); + + let mut sink = PostgresSink::new( + make_sink_config(&host, port), + SinkMode::Replication, + Some(SourceConnectionConfig::Mongodb { + // Never dialled: the drop path short-circuits for schemaless sources. + url: "mongodb://127.0.0.1:1/?connectTimeoutMS=100".into(), + }), + ) + .await + .unwrap(); + + // A full-row event that omits `nickname`, as such documents routinely do. + let event = CdcEvent { + lsn: Lsn(100), + timestamp_us: 1_700_000_000_000_000, + xid: 0, + table: TableId { + schema: "demo".into(), + name: "items".into(), + oid: 0, + }, + op: ChangeOp::Insert, + new: Some( + vec![ + ("_id".to_string(), ColumnValue::Text("item1".into())), + ("name".to_string(), ColumnValue::Text("Widget".into())), + ] + .into_iter() + .collect(), + ), + old: None, + primary_key_columns: vec!["_id".into()], + }; + + sink.write_batch(&[event]).await.unwrap(); + sink.flush().await.unwrap(); + + let exists: bool = client + .query_one( + "SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'items' + AND column_name = 'nickname' + )", + &[], + ) + .await + .unwrap() + .get(0); + + assert!( + exists, + "a field missing from one MongoDB batch must not drop the target column" + ); +} + +#[tokio::test] +#[ignore] +async fn test_cdc_mode_binary_column_round_trips() { + let (_container, host, port) = start_postgres().await; + tokio::time::sleep(Duration::from_secs(2)).await; + + let client = connect_client(&host, port).await; + client + .execute( + "CREATE TABLE \"public\".\"files\" ( + \"_cdc_op\" text, + \"_cdc_lsn\" bigint, + \"_cdc_timestamp_us\" bigint, + \"_cdc_snapshot\" boolean, + \"_cdc_schema\" text, + \"_cdc_table\" text, + \"_cdc_primary_key_columns\" text, + \"id\" text, + \"blob\" bytea, + \"_old_id\" text, + \"_old_blob\" bytea + )", + &[], + ) + .await + .unwrap(); + + let config = make_sink_config(&host, port); + let mut sink = PostgresSink::new(config, SinkMode::Cdc, None) + .await + .unwrap(); + + let payload = vec![0x00u8, 0x68, 0x69, 0xff]; + let event = make_insert_event( + "demo", + "files", + vec![ + ("id", ColumnValue::Text("f1".into())), + ("blob", ColumnValue::Bytes(payload.clone())), + ], + ); + + sink.write_batch(&[event]).await.unwrap(); + sink.flush().await.unwrap(); + + let stored: Vec = client + .query_one("SELECT \"blob\" FROM \"public\".\"files\"", &[]) + .await + .unwrap() + .get(0); + + assert_eq!(stored, payload); +}