From 8db1386364498c2d3111ea0b44beae8cba381e6c Mon Sep 17 00:00:00 2001 From: solutionsDigibull Date: Tue, 29 Sep 2026 11:54:24 +0530 Subject: [PATCH 1/3] Add DuckDB connector with execute_sql and search_objects support --- package.json | 1 + pnpm-lock.yaml | 96 +++++++++++++++ src/api/openapi.d.ts | 2 +- src/api/openapi.yaml | 2 +- src/config/toml-loader.ts | 90 +++++++------- src/connectors/duckdb/index.ts | 217 +++++++++++++++++++++++++++++++++ src/connectors/interface.ts | 3 +- src/index.ts | 1 + src/types/config.ts | 2 +- src/utils/allowed-keywords.ts | 2 + src/utils/dsn-obfuscate.ts | 17 ++- src/utils/error-classifier.ts | 1 + src/utils/parameter-mapper.ts | 7 +- src/utils/sql-parser.ts | 1 + 14 files changed, 385 insertions(+), 57 deletions(-) create mode 100644 src/connectors/duckdb/index.ts diff --git a/package.json b/package.json index a6b15c49..837806b8 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "author": "", "license": "MIT", "dependencies": { + "@duckdb/node-api": "1.5.6-r.1", "@iarna/toml": "^2.2.5", "@modelcontextprotocol/node": "^2.0.0", "@modelcontextprotocol/server": "^2.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7fa2030b..d848adf1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@duckdb/node-api': + specifier: 1.5.6-r.1 + version: 1.5.6-r.1 '@iarna/toml': specifier: ^2.2.5 version: 2.2.5 @@ -540,6 +543,52 @@ packages: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} + '@duckdb/node-api@1.5.6-r.1': + resolution: {integrity: sha512-yIPol6P4EQmsAUsb27donJf7iADrM31Ei/WkvhkZpSscjhEErYsuF19lsJ/qz5mUWxsKFQpLdjvrlhPQnMTBqw==} + + '@duckdb/node-bindings-darwin-arm64@1.5.6-r.1': + resolution: {integrity: sha512-KFT/J/pyouuAD1jftAIfcWVRdorg+c/yVR/BJL8LJKrST+mYC4d7QSuQ+iXSYbFpHENUAeZUagXrMPtZT8Xzsg==} + cpu: [arm64] + os: [darwin] + + '@duckdb/node-bindings-darwin-x64@1.5.6-r.1': + resolution: {integrity: sha512-tVwUZNvdQoaPJhrh3q2e6jxm1MhahNHJQK/pvCyco46NX0mEuGzlw9i0n2MArgsyzccdCwScOK9ctbyr+3oMbQ==} + cpu: [x64] + os: [darwin] + + '@duckdb/node-bindings-linux-arm64-musl@1.5.6-r.1': + resolution: {integrity: sha512-j6dcNLWnFvlQyKMtl36HyeDsOaetXFoj0dH21Q9MbLjABNuJs5ekDV6zMSEda/g5+z1Tm6r2KFOS8j3tGUCRiw==} + cpu: [arm64] + os: [linux] + + '@duckdb/node-bindings-linux-arm64@1.5.6-r.1': + resolution: {integrity: sha512-3jnUaDil1BrF/6vAZwVaInJ1xoBaWGHB4PDiUnzJWKCo8a8TLx9j7/gPyPl/UGRBftrYIuTihRZSfF4mWMgtWA==} + cpu: [arm64] + os: [linux] + + '@duckdb/node-bindings-linux-x64-musl@1.5.6-r.1': + resolution: {integrity: sha512-PVzIEZqp7QN07WBgjmIH0cK2GKXR735KmL6wnghTzRnhHWrePxx3MjZB9zZHI8tB73wdF8luarBm4EgqSjGyVA==} + cpu: [x64] + os: [linux] + + '@duckdb/node-bindings-linux-x64@1.5.6-r.1': + resolution: {integrity: sha512-IXcrZJE6kEqM0Z4wwdVnGBHX/frOS33VPNAwDXQ/Q+Fs84b058Qe94chrCCIcNJlG8Y3UHIUU4eBeCQxW5mN1Q==} + cpu: [x64] + os: [linux] + + '@duckdb/node-bindings-win32-arm64@1.5.6-r.1': + resolution: {integrity: sha512-nzNTZx9rus5QpuX+KH6UOPNeZp8yF25TxV6DiUly4RKgR5mu6xo8c7a7lS80AqxTJfJ9BwUIiqlCuYKWztRM5A==} + cpu: [arm64] + os: [win32] + + '@duckdb/node-bindings-win32-x64@1.5.6-r.1': + resolution: {integrity: sha512-5Rpn5WXEG5gY/u7Keh+c4kuwG/LiE31pCP9CoCohv+u0aFensk/yRIQVW/CXuIoITky9xFhdRzaWUtAnGTYxJQ==} + cpu: [x64] + os: [win32] + + '@duckdb/node-bindings@1.5.6-r.1': + resolution: {integrity: sha512-/pWlTyvKC1d16iSMJCBAPwlmzoeXRs5Y9cqG+TpKd/YAW6Ke05FJg099v/Im2bkS9BgznNyNcAK1NxA1eQhSUw==} + '@esbuild/aix-ppc64@0.25.5': resolution: {integrity: sha512-9o3TMmpmftaCMepOdA5k/yDw8SfInyzWWTjYTFCX3kPSDJMROQTb8jg+h9Cnwnmm1vOzvxN7gIfB5V2ewpjtGA==} engines: {node: '>=18'} @@ -1845,6 +1894,10 @@ packages: resolution: {integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==} engines: {node: '>=8'} + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} @@ -4122,6 +4175,47 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.9 + '@duckdb/node-api@1.5.6-r.1': + dependencies: + '@duckdb/node-bindings': 1.5.6-r.1 + + '@duckdb/node-bindings-darwin-arm64@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings-darwin-x64@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings-linux-arm64-musl@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings-linux-arm64@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings-linux-x64-musl@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings-linux-x64@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings-win32-arm64@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings-win32-x64@1.5.6-r.1': + optional: true + + '@duckdb/node-bindings@1.5.6-r.1': + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + '@duckdb/node-bindings-darwin-arm64': 1.5.6-r.1 + '@duckdb/node-bindings-darwin-x64': 1.5.6-r.1 + '@duckdb/node-bindings-linux-arm64': 1.5.6-r.1 + '@duckdb/node-bindings-linux-arm64-musl': 1.5.6-r.1 + '@duckdb/node-bindings-linux-x64': 1.5.6-r.1 + '@duckdb/node-bindings-linux-x64-musl': 1.5.6-r.1 + '@duckdb/node-bindings-win32-arm64': 1.5.6-r.1 + '@duckdb/node-bindings-win32-x64': 1.5.6-r.1 + '@esbuild/aix-ppc64@0.25.5': optional: true @@ -5543,6 +5637,8 @@ snapshots: detect-libc@2.0.4: {} + detect-libc@2.1.2: {} + diff@4.0.2: {} docker-compose@1.2.0: diff --git a/src/api/openapi.d.ts b/src/api/openapi.d.ts index da3ca37d..1ddfe964 100644 --- a/src/api/openapi.d.ts +++ b/src/api/openapi.d.ts @@ -64,7 +64,7 @@ export interface components { * @example postgres * @enum {string} */ - type: "postgres" | "mysql" | "mariadb" | "sqlserver" | "sqlite" | "oracle"; + type: "postgres" | "mysql" | "mariadb" | "sqlserver" | "sqlite" | "oracle" | "duckdb"; /** * @description Database host (not present for SQLite) * @example localhost diff --git a/src/api/openapi.yaml b/src/api/openapi.yaml index 293ed6fe..890b1a66 100644 --- a/src/api/openapi.yaml +++ b/src/api/openapi.yaml @@ -102,7 +102,7 @@ components: example: Production read replica for analytics queries type: type: string - enum: [postgres, mysql, mariadb, sqlserver, sqlite, oracle] + enum: [postgres, mysql, mariadb, sqlserver, sqlite, oracle,duckdb] description: Database type example: postgres host: diff --git a/src/config/toml-loader.ts b/src/config/toml-loader.ts index 8d285c49..eebc3fac 100644 --- a/src/config/toml-loader.ts +++ b/src/config/toml-loader.ts @@ -29,7 +29,7 @@ export function loadTomlConfig(): { sources: SourceConfig[]; tools?: TomlConfig[ if (!Array.isArray(parsedToml.sources)) { throw new Error( `Configuration file ${configPath}: must contain a [[sources]] array. ` + - `Use [[sources]] syntax for array of tables in TOML.` + `Use [[sources]] syntax for array of tables in TOML.` ); } @@ -89,7 +89,7 @@ function validateTomlConfig(config: TomlConfig, configPath: string): void { if (!config.sources) { throw new Error( `Configuration file ${configPath} must contain a [[sources]] array. ` + - `Example:\n\n[[sources]]\nid = "my_db"\ndsn = "postgres://..."` + `Example:\n\n[[sources]]\nid = "my_db"\ndsn = "postgres://..."` ); } @@ -98,7 +98,7 @@ function validateTomlConfig(config: TomlConfig, configPath: string): void { if (config.sources.length === 0) { throw new Error( `Configuration file ${configPath}: sources array cannot be empty. ` + - `Please define at least one source with [[sources]].` + `Please define at least one source with [[sources]].` ); } @@ -110,7 +110,7 @@ function validateTomlConfig(config: TomlConfig, configPath: string): void { if (!source.id) { throw new Error( `Configuration file ${configPath}: each source must have an 'id' field. ` + - `Example: [[sources]]\nid = "my_db"` + `Example: [[sources]]\nid = "my_db"` ); } @@ -124,7 +124,7 @@ function validateTomlConfig(config: TomlConfig, configPath: string): void { if (duplicates.length > 0) { throw new Error( `Configuration file ${configPath}: duplicate source IDs found: ${duplicates.join(", ")}. ` + - `Each source must have a unique 'id' field.` + `Each source must have a unique 'id' field.` ); } @@ -195,7 +195,7 @@ function validateToolsConfig( if (!isExecuteSql && (tool.readonly !== undefined || tool.max_rows !== undefined)) { throw new Error( `Configuration file ${configPath}: tool '${tool.name}' cannot have readonly or max_rows fields ` + - `(these are only valid for ${BUILTIN_TOOL_EXECUTE_SQL} tool)` + `(these are only valid for ${BUILTIN_TOOL_EXECUTE_SQL} tool)` ); } } else { @@ -311,8 +311,8 @@ function validateDSNFieldConflicts(source: SourceConfig, configPath: string): vo const conflict = (field: string, fieldValue: string, dsnValue: string): never => { throw new Error( `Configuration file ${configPath}: source '${source.id}' has conflicting ${field}: ` + - `the DSN specifies '${dsnValue}' but the ${field} field is '${fieldValue}'. ` + - `Set ${field} in only one place, or make the two values match.` + `the DSN specifies '${dsnValue}' but the ${field} field is '${fieldValue}'. ` + + `Set ${field} in only one place, or make the two values match.` ); }; @@ -358,7 +358,7 @@ function validateDSNFieldConflicts(source: SourceConfig, configPath: string): vo if (source.database && !info.database) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has a 'database' field but the DSN names no database. ` + - `The field is ignored at connection time — add the database to the DSN, or use individual connection parameters instead of a DSN.` + `The field is ignored at connection time — add the database to the DSN, or use individual connection parameters instead of a DSN.` ); } if (source.user && info.user && source.user !== info.user) { @@ -372,12 +372,12 @@ function validateDSNFieldConflicts(source: SourceConfig, configPath: string): vo if (!url.password) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has a 'password' field but the DSN has no password. ` + - `The field is ignored at connection time — add the password to the DSN, or use individual connection parameters instead of a DSN.` + `The field is ignored at connection time — add the password to the DSN, or use individual connection parameters instead of a DSN.` ); } throw new Error( `Configuration file ${configPath}: source '${source.id}' has a 'password' field that conflicts ` + - `with the password in the DSN. Set the password in only one place.` + `with the password in the DSN. Set the password in only one place.` ); } @@ -433,19 +433,19 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (!source.dsn && !hasConnectionParams) { throw new Error( `Configuration file ${configPath}: source '${source.id}' must have either:\n` + - ` - 'dsn' field (e.g., dsn = "postgres://user:pass@host:5432/dbname")\n` + - ` - OR connection parameters (type, host, database, user, password)\n` + - ` - For SQLite: type = "sqlite" and database path` + ` - 'dsn' field (e.g., dsn = "postgres://user:pass@host:5432/dbname")\n` + + ` - OR connection parameters (type, host, database, user, password)\n` + + ` - For SQLite: type = "sqlite" and database path` ); } // Validate type if provided if (source.type) { - const validTypes = ["postgres", "mysql", "mariadb", "sqlserver", "sqlite", "oracle"]; + const validTypes = ["postgres", "mysql", "mariadb", "sqlserver", "sqlite", "oracle", "duckdb"]; if (!validTypes.includes(source.type)) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid type '${source.type}'. ` + - `Valid types: ${validTypes.join(", ")}` + `Valid types: ${validTypes.join(", ")}` ); } } @@ -457,7 +457,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { ) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid aws_iam_auth. ` + - `Must be a boolean (true or false).` + `Must be a boolean (true or false).` ); } @@ -468,7 +468,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { ) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid aws_region. ` + - `Must be a non-empty string (e.g., "eu-west-1").` + `Must be a non-empty string (e.g., "eu-west-1").` ); } } @@ -480,13 +480,13 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { ) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid aws_profile. ` + - `Must be a non-empty string.` + `Must be a non-empty string.` ); } if (source.aws_iam_auth !== true) { throw new Error( `Configuration file ${configPath}: source '${source.id}' aws_profile requires ` + - `aws_iam_auth = true.` + `aws_iam_auth = true.` ); } } @@ -496,13 +496,13 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (!source.type || !validIamTypes.includes(source.type)) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has aws_iam_auth enabled, ` + - `but this is only supported for postgres, mysql, and mariadb sources.` + `but this is only supported for postgres, mysql, and mariadb sources.` ); } if (!source.aws_region) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has aws_iam_auth enabled ` + - `but aws_region is not specified.` + `but aws_region is not specified.` ); } } @@ -512,7 +512,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (typeof source.connection_timeout !== "number" || source.connection_timeout <= 0) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid connection_timeout. ` + - `Must be a positive number (in seconds).` + `Must be a positive number (in seconds).` ); } } @@ -522,7 +522,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (typeof source.query_timeout !== "number" || source.query_timeout <= 0) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid query_timeout. ` + - `Must be a positive number (in seconds).` + `Must be a positive number (in seconds).` ); } } @@ -540,7 +540,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { ) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid pool_max_connections. ` + - `Must be an integer between 1 and 1000.` + `Must be an integer between 1 and 1000.` ); } } @@ -554,7 +554,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { ) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid ssh_port. ` + - `Must be between 1 and 65535.` + `Must be between 1 and 65535.` ); } } @@ -565,7 +565,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (source.type === "sqlite") { throw new Error( `Configuration file ${configPath}: source '${source.id}' has sslmode but SQLite does not support SSL. ` + - `Remove the sslmode field for SQLite sources.` + `Remove the sslmode field for SQLite sources.` ); } @@ -573,7 +573,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (!validSslModes.includes(source.sslmode)) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid sslmode '${source.sslmode}'. ` + - `Valid values: ${validSslModes.join(", ")}` + `Valid values: ${validSslModes.join(", ")}` ); } @@ -590,7 +590,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { const supported = ["disable", "require", ...(verifyModesByType[source.type] ?? [])]; throw new Error( `Configuration file ${configPath}: source '${source.id}' has sslmode '${source.sslmode}' which is not supported for ${source.type}. ` + - `Valid values for ${source.type}: ${supported.join(", ")}` + `Valid values for ${source.type}: ${supported.join(", ")}` ); } } @@ -617,7 +617,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (source.sslmode !== "verify-ca" && source.sslmode !== "verify-full") { throw new Error( `Configuration file ${configPath}: source '${source.id}' has sslrootcert but sslmode is '${source.sslmode ?? "not set"}'. ` + - `sslrootcert requires sslmode 'verify-ca' or 'verify-full'` + `sslrootcert requires sslmode 'verify-ca' or 'verify-full'` ); } @@ -636,8 +636,8 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (source.sslcert === undefined || source.sslkey === undefined) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has ${source.sslcert !== undefined ? "sslcert" : "sslkey"} ` + - `without ${source.sslcert !== undefined ? "sslkey" : "sslcert"}. ` + - `sslcert and sslkey must be set together for client certificate authentication` + `without ${source.sslcert !== undefined ? "sslkey" : "sslcert"}. ` + + `sslcert and sslkey must be set together for client certificate authentication` ); } // libpq sends the client certificate in every SSL mode, but node-postgres @@ -646,7 +646,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (!CLIENT_CERT_SSL_MODES.includes(source.sslmode ?? "")) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has sslcert/sslkey but sslmode is '${source.sslmode ?? "not set"}'. ` + - `sslcert/sslkey require sslmode 'require', 'verify-ca' or 'verify-full'` + `sslcert/sslkey require sslmode 'require', 'verify-ca' or 'verify-full'` ); } validateReadableFile(source, "sslcert", source.sslcert, configPath); @@ -667,7 +667,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (!validAuthMethods.includes(source.authentication)) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid authentication '${source.authentication}'. ` + - `Valid values: ${validAuthMethods.join(", ")}` + `Valid values: ${validAuthMethods.join(", ")}` ); } @@ -692,13 +692,13 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (source.authentication === undefined) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has domain but authentication is not set. ` + - `Add authentication = "ntlm" to use Windows domain authentication.` + `Add authentication = "ntlm" to use Windows domain authentication.` ); } if (source.authentication !== "ntlm") { throw new Error( `Configuration file ${configPath}: source '${source.id}' has domain but authentication is set to '${source.authentication}'. ` + - `Domain is only valid with authentication = "ntlm".` + `Domain is only valid with authentication = "ntlm".` ); } } @@ -713,7 +713,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (typeof source.search_path !== "string" || source.search_path.trim().length === 0) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid search_path. ` + - `Must be a non-empty string of comma-separated schema names (e.g., "myschema,public").` + `Must be a non-empty string of comma-separated schema names (e.g., "myschema,public").` ); } @@ -736,7 +736,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { ) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid timezone '${source.timezone}'. ` + - `Must be "local", "Z" (UTC), or an offset like "+09:00".` + `Must be "local", "Z" (UTC), or an offset like "+09:00".` ); } } @@ -755,7 +755,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (typeof source.charset !== "string" || source.charset.trim() === "") { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid charset '${source.charset}'. ` + - `Must be a non-empty string naming a character set (e.g. "utf8mb4").` + `Must be a non-empty string naming a character set (e.g. "utf8mb4").` ); } } @@ -772,7 +772,7 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if (typeof source.collation !== "string" || source.collation.trim() === "") { throw new Error( `Configuration file ${configPath}: source '${source.id}' has invalid collation '${source.collation}'. ` + - `Must be a non-empty string naming a collation (e.g. "utf8mb4_0900_ai_ci").` + `Must be a non-empty string naming a collation (e.g. "utf8mb4_0900_ai_ci").` ); } } @@ -781,13 +781,13 @@ function validateSourceConfig(source: SourceConfig, configPath: string): void { if ((source as any).readonly !== undefined) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has 'readonly' field, but readonly must be configured per-tool, not per-source. ` + - `Move 'readonly' to [[tools]] configuration instead.` + `Move 'readonly' to [[tools]] configuration instead.` ); } if ((source as any).max_rows !== undefined) { throw new Error( `Configuration file ${configPath}: source '${source.id}' has 'max_rows' field, but max_rows must be configured per-tool, not per-source. ` + - `Move 'max_rows' to [[tools]] configuration instead.` + `Move 'max_rows' to [[tools]] configuration instead.` ); } } @@ -1088,14 +1088,14 @@ export function buildDSNFromSource(source: SourceConfig): string { if (!source.host || !source.user || !source.database) { throw new Error( `Source '${source.id}': missing required connection parameters. ` + - `Required: type, host, user, database` + `Required: type, host, user, database` ); } if (passwordRequired && !source.password) { throw new Error( `Source '${source.id}': password is required. ` + - `(Password is optional for azure-active-directory-access-token authentication ` + - `or when aws_iam_auth=true)` + `(Password is optional for azure-active-directory-access-token authentication ` + + `or when aws_iam_auth=true)` ); } diff --git a/src/connectors/duckdb/index.ts b/src/connectors/duckdb/index.ts new file mode 100644 index 00000000..61108368 --- /dev/null +++ b/src/connectors/duckdb/index.ts @@ -0,0 +1,217 @@ +// src/connectors/duckdb/index.ts +import { DuckDBInstance, DuckDBConnection } from "@duckdb/node-api"; +import { + Connector, + ConnectorType, + DSNParser, + ConnectorConfig, + ExecuteOptions, + SQLResult, + TableColumn, + TableIndex, + StoredProcedure, + ConnectorRegistry, +} from "../interface.js"; +import { obfuscateDSNPassword } from "../../utils/dsn-obfuscate.js"; + +// ------------------------------------------------------------ +// DSN Parser +// ------------------------------------------------------------ +class DuckDBDSNParser implements DSNParser { + async parse(dsn: string, config?: ConnectorConfig): Promise<{ dbPath: string }> { + if (!this.isValidDSN(dsn)) { + const obfuscatedDSN = obfuscateDSNPassword(dsn); + throw new Error( + `Invalid DuckDB DSN: ${obfuscatedDSN}\nExpected format: ${this.getSampleDSN()}` + ); + } + + const pathMatch = dsn.match(/^duckdb:\/\/(.+)$/); + if (!pathMatch) throw new Error("Could not extract database path from DSN"); + + let dbPath = pathMatch[1]; + // Windows drive paths: strip leading "/" so DuckDB does not treat it as UNC + if (/^\/[A-Za-z]:[\\/]/.test(dbPath)) { + dbPath = dbPath.slice(1); + } + return { dbPath }; + } + + getSampleDSN(): string { + return "duckdb:///path/to/database.duckdb"; + } + + isValidDSN(dsn: string): boolean { + return /^duckdb:\/\/(.+)\.duckdb$/.test(dsn) || dsn === "duckdb://:memory:"; + } +} + +// ------------------------------------------------------------ +// Connector +// ------------------------------------------------------------ +class DuckDBConnector implements Connector { + id: ConnectorType = "duckdb"; + name = "DuckDB"; + dsnParser = new DuckDBDSNParser(); + + private instance: DuckDBInstance | null = null; + private connection: DuckDBConnection | null = null; + private sourceId: string = "default"; + + getId(): string { + return this.sourceId; + } + + clone(): Connector { + return new DuckDBConnector(); + } + + async connect(dsn: string, initScript?: string, config?: ConnectorConfig): Promise { + const parsed = await this.dsnParser.parse(dsn, config); + this.instance = await DuckDBInstance.fromCache(parsed.dbPath); + this.connection = await this.instance.connect(); + if (initScript) { + await this.connection.run(initScript); + } + } + + async disconnect(): Promise { + if (this.connection) { + this.connection.closeSync(); + this.connection = null; + } + this.instance = null; + } + + async getSchemas(): Promise { + const res = await this.connection!.run( + "SELECT schema_name FROM information_schema.schemata" + ); + return res.getRowObjectsJS().map((r) => r.schema_name as string); + } + + async getTables(schema?: string): Promise { + const target = schema ?? "main"; + const res = await this.connection!.run( + `SELECT table_name FROM information_schema.tables + WHERE table_schema = '${target}'` + ); + return res.getRowObjectsJS().map((r) => r.table_name as string); + } + + async getViews(schema?: string): Promise { + const target = schema ?? "main"; + const res = await this.connection!.run( + `SELECT table_name FROM information_schema.views + WHERE table_schema = '${target}'` + ); + return res.getRowObjectsJS().map((r) => r.table_name as string); + } + + async getTableSchema(tableName: string, schema?: string): Promise { + const target = schema ?? "main"; + const res = await this.connection!.run( + `SELECT column_name, data_type, is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = '${target}' AND table_name = '${tableName}' + ORDER BY ordinal_position` + ); + return res.getRowObjectsJS().map((r) => ({ + column_name: r.column_name as string, + data_type: r.data_type as string, + is_nullable: r.is_nullable as string, + column_default: (r.column_default as string | null) ?? null, + description: null, + })); + } + + async tableExists(tableName: string, schema?: string): Promise { + const target = schema ?? "main"; + const res = await this.connection!.run( + `SELECT COUNT(*) AS cnt FROM information_schema.tables + WHERE table_schema = '${target}' AND table_name = '${tableName}'` + ); + const rows = res.getRowObjectsJS(); + return rows.length > 0 && Number(rows[0].cnt) > 0; + } + + async getTableIndexes(tableName: string, schema?: string): Promise { + const target = schema ?? "main"; + const res = await this.connection!.run( + `SELECT index_name, is_unique, is_primary, expressions + FROM duckdb_indexes() + WHERE schema_name = '${target}' AND table_name = '${tableName}'` + ); + return res.getRowObjectsJS().map((r) => ({ + index_name: r.index_name as string, + column_names: (r.expressions as string[]) ?? [], + is_unique: Boolean(r.is_unique), + is_primary: Boolean(r.is_primary), + })); + } + + async getStoredProcedures( + _schema?: string, + _routineType?: "procedure" | "function" + ): Promise { + return []; + } + + async getStoredProcedureDetail( + _procedureName: string, + _schema?: string + ): Promise { + throw new Error("DuckDB does not support stored procedures"); + } + + async getTableRowCount(tableName: string, schema?: string): Promise { + const target = schema ?? "main"; + const res = await this.connection!.run( + `SELECT estimated_size FROM duckdb_tables() + WHERE schema_name = '${target}' AND table_name = '${tableName}'` + ); + const rows = res.getRowObjectsJS(); + return rows.length ? Number(rows[0].estimated_size) : null; + } + + async getTableComment(_tableName: string, _schema?: string): Promise { + return null; + } + + async executeSQL( + sql: string, + options: ExecuteOptions, + parameters?: any[] + ): Promise { + let result; + if (parameters?.length) { + const prepared = await this.connection!.prepare(sql); + result = await prepared.run(...parameters); + } else { + result = await this.connection!.run(sql); + } + + const allRows = await result.getRowObjectsJS(); + + const rows = options.maxRows + ? allRows.slice(0, options.maxRows) + : allRows; + + return { + resultSets: [ + { + rows, + rowCount: rows.length, + }, + ], + }; + } +} + +// ------------------------------------------------------------ +// Self-registration +// ------------------------------------------------------------ +const duckdbConnector = new DuckDBConnector(); +ConnectorRegistry.register(duckdbConnector); + +export { DuckDBConnector, DuckDBDSNParser }; \ No newline at end of file diff --git a/src/connectors/interface.ts b/src/connectors/interface.ts index ff93cf05..e89634d3 100644 --- a/src/connectors/interface.ts +++ b/src/connectors/interface.ts @@ -1,7 +1,8 @@ /** * Type definition for supported database connector types */ -export type ConnectorType = "postgres" | "mysql" | "mariadb" | "sqlite" | "sqlserver" | "oracle"; +// export type ConnectorType = "postgres" | "mysql" | "mariadb" | "sqlite" | "sqlserver" | "oracle"; +export type ConnectorType = "postgres" | "mysql" | "mariadb" | "sqlite" | "sqlserver" | "oracle" | "duckdb"; /** * Database Connector Interface diff --git a/src/index.ts b/src/index.ts index 62cb627d..c96c7716 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ const connectorModules = [ { load: () => import("./connectors/mysql/index.js"), name: "MySQL", driver: "mysql2" }, { load: () => import("./connectors/mariadb/index.js"), name: "MariaDB", driver: "mariadb" }, { load: () => import("./connectors/oracle/index.js"), name: "Oracle", driver: "oracledb" }, + { load: () => import("./connectors/duckdb/index.js"), name: "DuckDB", driver: "@duckdb/node-api" }, ]; loadConnectors(connectorModules) diff --git a/src/types/config.ts b/src/types/config.ts index 6f2a3459..c2138e01 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -27,7 +27,7 @@ export interface SSHConfig { * Database connection parameters (alternative to DSN) */ export interface ConnectionParams { - type: "postgres" | "mysql" | "mariadb" | "sqlserver" | "sqlite" | "oracle"; + type: "postgres" | "mysql" | "mariadb" | "sqlserver" | "sqlite" | "oracle" | "duckdb"; host?: string; port?: number; database?: string; diff --git a/src/utils/allowed-keywords.ts b/src/utils/allowed-keywords.ts index 0c9be7e2..e97f5a6d 100644 --- a/src/utils/allowed-keywords.ts +++ b/src/utils/allowed-keywords.ts @@ -19,6 +19,7 @@ export const allowedKeywords: Record = { // DBMS_XPLAN (see OracleConnector.explainQuery). EXPLAIN PLAN only parses // the statement, it never executes it. oracle: ["select", "with", "explain"], + duckdb: ["select", "with", "explain", "pragma"] }; /** @@ -219,6 +220,7 @@ const mutatingPatterns: Record = { sqlite: mutatingPatternWithReplace, sqlserver: mutatingPatternSqlServer, oracle: mutatingPattern, + duckdb: /^\s*(insert|update|delete|merge|create|drop|alter|truncate|replace|attach|detach|copy|export|import|install|load|pragma)\b/i, }; /** diff --git a/src/utils/dsn-obfuscate.ts b/src/utils/dsn-obfuscate.ts index a440f6c7..2fc8f4b5 100644 --- a/src/utils/dsn-obfuscate.ts +++ b/src/utils/dsn-obfuscate.ts @@ -1,6 +1,7 @@ import type { SSHTunnelConfig } from '../types/ssh.js'; import type { ConnectorType } from '../connectors/interface.js'; import { SafeURL } from './safe-url.js'; +import duckdb from '@duckdb/node-api'; /** * Parsed connection information from a DSN string @@ -54,6 +55,10 @@ export function parseConnectionInfoFromDSN(dsn: string): ParsedConnectionInfo | return { type }; } + if (type === 'duckdb') { + return { type }; + } + // Parse other database DSNs using SafeURL const url = new SafeURL(dsn); @@ -168,19 +173,19 @@ export function obfuscateSSHConfig(config: SSHTunnelConfig): Partial> = { // ORA-01017: invalid username/password; ORA-28000: account locked oracle: ["ORA-01017", "ORA-28000"], sqlite: [], // no network/auth layer + duckdb: [], // no network/auth layer }; function unreachableMessage(sourceId: string): string { diff --git a/src/utils/parameter-mapper.ts b/src/utils/parameter-mapper.ts index 928d96f1..3401cf0f 100644 --- a/src/utils/parameter-mapper.ts +++ b/src/utils/parameter-mapper.ts @@ -17,6 +17,7 @@ export const PARAMETER_STYLES = { sqlserver: "named", // @p1, @p2, @p3 sqlite: "positional", // ?, ?, ? oracle: "colon", // :1, :2, :3 + duckdb: "?", // DuckDB uses ? positional placeholders, same as SQLite } as const; /** @@ -86,8 +87,8 @@ export function validateParameterStyle( throw new Error( `Invalid parameter syntax for ${connectorType}. ` + - `Expected ${expectedStyle} style (${examples[expectedStyle]}), ` + - `but found ${detectedStyle} style in statement.` + `Expected ${expectedStyle} style (${examples[expectedStyle]}), ` + + `but found ${detectedStyle} style in statement.` ); } } @@ -171,7 +172,7 @@ export function validateParameters( if (paramCount !== definedCount) { throw new Error( `Parameter count mismatch: SQL statement has ${paramCount} parameter(s), ` + - `but ${definedCount} parameter(s) defined in tool configuration.` + `but ${definedCount} parameter(s) defined in tool configuration.` ); } } diff --git a/src/utils/sql-parser.ts b/src/utils/sql-parser.ts index 33ba041d..f6950084 100644 --- a/src/utils/sql-parser.ts +++ b/src/utils/sql-parser.ts @@ -217,6 +217,7 @@ const dialectScanners: Record = { mysql: scanTokenMySQL, mariadb: scanTokenMySQL, sqlite: scanTokenSQLite, + duckdb: scanTokenSQLite, sqlserver: scanTokenSQLServer, oracle: scanTokenOracle, }; From ee4e1f87f866af64989aa1beddd4da660574ccb5 Mon Sep 17 00:00:00 2001 From: "solutions@Digibull" Date: Wed, 30 Sep 2026 11:37:07 +0530 Subject: [PATCH 2/3] Added Files Added toml and implementation of the TXT to sql via MCP tools registered in bifrost --- DBHub_MCP_TXTtoSQL.md | 395 ++++++++++++++++++++++++++++++++++++++++++ dbhub.toml | 12 ++ 2 files changed, 407 insertions(+) create mode 100644 DBHub_MCP_TXTtoSQL.md create mode 100644 dbhub.toml diff --git a/DBHub_MCP_TXTtoSQL.md b/DBHub_MCP_TXTtoSQL.md new file mode 100644 index 00000000..fa61106e --- /dev/null +++ b/DBHub_MCP_TXTtoSQL.md @@ -0,0 +1,395 @@ +# DBHub + DuckDB + Bifrost + Ollama — Handoff Document + +**Date:** 2026-09-30 +**Owner:** DigiBull / solutionsDigibull +**Status:** Working end-to-end, ready for team handoff +**Repos:** https://github.com/solutionsDigibull/dbhub (branch `duckdb-connector`, commit `8db1386`) + +--- + +## 1. What Was Built + +A fully local, no-API MCP pipeline that lets a local LLM query **PostgreSQL** and **DuckDB** simultaneously through a single MCP endpoint. + +``` +┌─────────────────┐ ┌──────────────┐ ┌─────────────────────┐ +│ MCP Client / │────▶│ Bifrost │────▶│ DBHub │ +│ Chat UI │ │ (Docker) │ │ (Windows host) │ +│ │◀────│ :8010 │◀────│ :8020 │ +└─────────────────┘ └──────┬───────┘ └──────────┬──────────┘ + │ │ + ▼ ├──▶ PostgreSQL + ┌──────────────┐ │ 192.168.1.18:5432 + │ Ollama │ │ logic_db_2 + │ (Windows) │ │ + │ :11434 │ └──▶ DuckDB + │ qwen2.5- │ C:/SRIM-DB-SETUP/ + │ coder:7b │ Databases/ + └──────────────┘ Bom_storage.duckdb +``` + +**Why this design:** +- DBHub has no native DuckDB connector — we built one and patched it into a private fork. +- Bifrost is the MCP gateway + LLM router. Docker-hosted, so it needs `host.docker.internal` to reach Windows host services. +- Ollama runs on Windows host, bound to `0.0.0.0:11434` so the Docker container can reach it. + +--- + +## 2. Deliverables + +| Artifact | Location | Purpose | +|---|---|---| +| DBHub fork | `github.com/solutionsDigibull/dbhub` branch `duckdb-connector` | Custom DuckDB connector | +| `duckdb/index.ts` | `src/connectors/duckdb/index.ts` | The connector implementation | +| `dbhub.toml` | repo root | Multi-source configuration | +| Bifrost provider config | Bifrost API / UI | Ollama provider | +| Bifrost MCP client config | Bifrost API / UI | `DBHub_MCP` client | +| This document | repo / wiki | Handoff + reproduction | + +--- + +## 3. DBHub Fork — Files Changed (14 total) + +### 3.1 New File: `src/connectors/duckdb/index.ts` + +The complete connector. Key features: + +- **DSN parser** accepts `duckdb:///C:/path/file.duckdb` and strips the leading `/` on Windows drive letters (DuckDB otherwise treats `/C:/...` as a UNC path). +- **`connect`** uses `DuckDBInstance.fromCache()` to avoid attaching the same file twice in one process. +- **`executeSQL`** returns `{ resultSets: [{ rows, rowCount }] }` — matches `SQLResultSet` in `interface.ts`. +- Uses `result.getRowObjectsJS()` to convert DuckDB types (BIGINT, DECIMAL, DATE, TIMESTAMP, LIST, STRUCT) to native JS values. +- Implements all required interface methods: `getSchemas`, `getTables`, `getViews`, `getTableSchema`, `tableExists`, `getTableIndexes`, `getStoredProcedures`, `getStoredProcedureDetail`, `executeSQL`, plus optional `getTableRowCount`, `getTableComment`. + +### 3.2 Modified Files — Add `"duckdb"` to Every Type Union + +| File | Change | +|---|---| +| `src/connectors/interface.ts` | `ConnectorType` union += `"duckdb"` | +| `src/types/config.ts` | `SourceConfig.type` union += `"duckdb"` | +| `src/utils/dsn-obfuscate.ts` | `protocolToConnectorType` map += `duckdb: "duckdb"`; `ports` map += `duckdb: undefined`; `parseConnectionInfoFromDSN` short-circuits DuckDB like SQLite | +| `src/config/toml-loader.ts` | `validTypes` array += `"duckdb"` | +| `src/utils/allowed-keywords.ts` | `allowedKeywords` and `mutatingPatterns` records += DuckDB entries (copy from SQLite) | +| `src/utils/sql-parser.ts` | `dialectScanners` record += `duckdb: sqliteScanner` | +| `src/utils/error-classifier.ts` | `AUTH_CODES` record += `duckdb: []` | +| `src/utils/parameter-mapper.ts` | `PARAMETER_STYLES` += `duckdb: "?"` | +| `src/api/openapi.yaml` | `DataSource.type` enum += `duckdb`; regenerate with `pnpm run generate:api-types` | + +### 3.3 Modified File — `src/index.ts` + +Add DuckDB to the connector loader array: + +```typescript +{ load: () => import("./connectors/duckdb/index.js"), + name: "DuckDB", + driver: "@duckdb/node-api" }, +``` + +### 3.4 Build Changes + +- `package.json` → add dependency `@duckdb/node-api` (installed with `pnpm add @duckdb/node-api -w`) +- `pnpm-lock.yaml` — updated automatically + +--- + +## 4. Runtime Configuration + +### 4.1 `dbhub.toml` + +```toml +[[sources]] +id = "postgres_logic" +description = "Production PostgreSQL database" +dsn = "postgresql://postgres:DigiBull@192.168.1.18:5432/logic_db_2?sslmode=disable" + +[[sources]] +id = "duckdb_local" +description = "Local DuckDB analytical database" +dsn = "duckdb:///C:/SRIM-DB-SETUP/Databases/Bom_storage.duckdb" +``` + +### 4.2 DBHub Start Command + +**Critical:** `--allowed-hosts` must include `host.docker.internal` — DBHub rejects any request whose `Host` header is not on its allow-list with HTTP 403. + +```powershell +node dist/index.js ` + --transport http ` + --port 8020 ` + --config ./dbhub.toml ` + --allowed-hosts "host.docker.internal" +``` + +To make this permanent, set the env var before starting: + +```powershell +$env:DBHUB_ALLOWED_HOSTS = "host.docker.internal" +``` + +### 4.3 Ollama on Windows + +Bind to all interfaces so Docker can reach it: + +```powershell +[System.Environment]::SetEnvironmentVariable("OLLAMA_HOST", "0.0.0.0:11434", "User") +``` + +Restart Ollama. Verify from Docker: + +```powershell +docker exec -it bifrost wget -qO- http://host.docker.internal:11434/api/tags +``` + +### 4.4 Bifrost — Ollama Provider (Two-Step Since v1.5.0) + +**Step 1 — Create provider without keys:** + +```json +POST /api/providers +{ + "provider": "ollama", + "network_config": { + "base_url": "http://host.docker.internal:11434", + "default_request_timeout_in_seconds": 300, + "allow_private_network": true + } +} +``` + +`allow_private_network: true` is required — Bifrost blocks private IPs by default. + +**Step 2 — Add the key separately:** + +```json +POST /api/providers/ollama/keys +{ + "name": "ollama-local", + "value": "ollama", + "models": ["*"], + "weight": 1.0, + "ollama_key_config": { + "url": "http://host.docker.internal:11434" + } +} +``` + +Do **not** embed the `keys` array in the provider creation payload — v1.5.0 ignores it silently. + +### 4.5 Bifrost — DBHub MCP Client + +```json +POST /api/mcp/client +{ + "name": "DBHub_MCP", + "connection_type": "http", + "connection_string": "http://host.docker.internal:8020/mcp", + "auth_type": "none", + "tools_to_execute": ["*"] +} +``` + +Then set `allow_on_all_virtual_keys: true` and `allow_by_default: true` via `PUT /api/mcp/client/{id}` so the tools are visible without a Virtual Key. + +### 4.6 Bifrost — Access Control + +| Setting | Where | Value | +|---|---|---| +| `tools_to_execute` | MCP client | `["*"]` | +| `allow_by_default` | MCP client | `true` | +| `allow_on_all_virtual_keys` | MCP client | `true` | +| Virtual Key MCP configs | If used | Add `DBHub_MCP` with `["*"]` | + +Since Bifrost v1.5.0, an empty Virtual Key `mcp_configs` array means **deny all** (previously "allow all"). This caught us during setup. + +--- + +## 5. Verification Commands + +Run these in order to confirm each layer is working. + +### 5.1 DBHub Serving Tools Directly + +```powershell +curl.exe -X POST http://localhost:8020/mcp ` + -H "Content-Type: application/json" ` + -H "Accept: application/json, text/event-stream" ` + -d '{\"jsonrpc\":\"2.0\",\"method\":\"tools/list\",\"id\":1}' +``` + +Expected: SSE response with 4 tools. + +### 5.2 Bifrost Reaches DBHub + +```powershell +docker exec -it bifrost wget -qO- \ + --header='Accept: application/json, text/event-stream' \ + --header='Content-Type: application/json' \ + http://host.docker.internal:8020/mcp +``` + +Expected: no 403. + +### 5.3 Bifrost Discovers 4 Tools + +```powershell +$headers = @{ + "Authorization" = "Bearer " + "Content-Type" = "application/json" +} +$clients = Invoke-RestMethod -Uri "http://localhost:8010/api/mcp/clients" -Headers $headers +$clients.clients | Where-Object { $_.config.name -eq "DBHub_MCP" } | + Select-Object name, state, @{N='Tools';E={$_.tools.Count}} +``` + +Expected: `Tools: 4`. + +### 5.4 Ollama Reachable from Bifrost + +```powershell +Invoke-RestMethod -Uri "http://localhost:8010/v1/models" -Headers $headers | + Select-Object -ExpandProperty data | Select-Object id +``` + +Expected: `ollama/qwen2.5-coder:7b`, `ollama/qwen3:8b`, etc. + +--- + +## 6. End-to-End Query Flow (The Pattern to Reuse) + +Due to `qwen2.5-coder:7b` not emitting native `tool_calls`, use this **4-step manual pattern**: + +```powershell +$headers = @{ + "Authorization" = "Bearer " + "Content-Type" = "application/json" + "x-bf-mcp-include-clients" = "DBHub_MCP" # scopes tools to DBHub only +} + +# 1. Ask the model +$chat = Invoke-RestMethod -Uri "http://localhost:8010/v1/chat/completions" ` + -Method Post -Headers $headers -Body (@{ + model = "ollama/qwen2.5-coder:7b" + messages = @( @{ role = "user"; content = "Show me 3 rows from gerber_checklist" } ) + } | ConvertTo-Json -Depth 5) + +# 2. Parse the model's JSON output (emitted as content, not tool_calls) +$call = $chat.choices[0].message.content | ConvertFrom-Json + +# 3. Execute through Bifrost +$result = Invoke-RestMethod -Uri "http://localhost:8010/v1/mcp/tool/execute" ` + -Method Post -Headers $headers -Body (@{ + id = "call_auto_1" + type = "function" + function = @{ + name = $call.name + arguments = ($call.arguments | ConvertTo-Json -Compress) + } + } | ConvertTo-Json -Depth 5) + +# 4. Feed result back to model for natural-language answer (optional) +$answer = Invoke-RestMethod -Uri "http://localhost:8010/v1/chat/completions" ` + -Method Post -Headers $headers -Body (@{ + model = "ollama/qwen2.5-coder:7b" + messages = @( + @{ role = "user"; content = "Show me 3 rows from gerber_checklist" } + @{ role = "assistant"; content = $null; tool_calls = @(@{ + id = "call_auto_1"; type = "function" + function = @{ name = $call.name; arguments = ($call.arguments | ConvertTo-Json -Compress) } + })} + @{ role = "tool"; tool_call_id = "call_auto_1"; content = $result.content } + ) + } | ConvertTo-Json -Depth 10) + +$answer.choices[0].message.content +``` + +**Why the `x-bf-mcp-include-clients` header matters:** Without it, Bifrost injects tools from all MCP clients (distributor, intake, gerber, etc.), and small models pick the wrong tool. Scoping to `DBHub_MCP` drops the tool count from ~20 to 4 and dramatically improves accuracy. + +--- + +## 7. Known Limitations + +| Limitation | Impact | Workaround | +|---|---|---| +| `qwen2.5-coder:7b` emits tool calls as text, not native `tool_calls` | Bifrost cannot auto-execute; manual parsing required | Try `qwen3:8b`; or keep the manual pattern | +| Bifrost in Docker cannot reach Windows `localhost` | All URLs must use `host.docker.internal` | Standard Docker networking | +| DBHub rejects unknown `Host` headers with 403 | Bifrost discovery fails silently (Tools: 0) | `--allowed-hosts host.docker.internal` | +| Fork diverges from upstream DBHub | Future `git pull` on main will conflict on ~10 files | Rebase `duckdb-connector` and re-add `duckdb` to type unions | +| DuckDB `getStoredProcedures` returns `[]` | `search_objects` with `object_type: "procedure"` returns nothing | DuckDB has no stored procedures; use macros | +| `getTableIndexes` returns expressions, not column names | `column_names` field contains the SQL expression | Parse if exact names required | + +--- + +## 8. Dev Team Next Steps + +### 8.1 Immediate (Today) + +1. `git clone https://github.com/solutionsDigibull/dbhub.git` +2. `git checkout duckdb-connector` +3. `pnpm install && pnpm exec tsup` (skip `pnpm build` — frontend has missing deps) +4. Verify `dist/duckdb-*.js` exists +5. Start DBHub with the `--allowed-hosts` flag +6. Run verification commands in Section 5 + +### 8.2 Short-Term (This Week) + +- Try `qwen3:8b` for native tool_calls — if it works, remove the manual parse step +- Add a system prompt that lists the four DBHub tools to help smaller models +- Write a reusable `Invoke-BifrostTool` PowerShell function for the team +- Add a `Makefile` or `run.ps1` that starts DBHub with the correct flags every time +- Set up an internal npm package `@digibull/dbhub` from the fork's CI + +### 8.3 Medium-Term (This Month) + +- **Upstream the DuckDB connector** to `bytebase/dbhub` as a Pull Request. The implementation follows their documented `Connector` interface; the change is isolated to ~10 type-union edits + one new file. This would eliminate the maintenance burden. +- Add DuckDB `getTableIndexes` column extraction (parse expressions → names) +- Add DuckDB types tests for BIGINT, DECIMAL, DATE, TIMESTAMP, LIST, STRUCT +- Add a hot-reload test for `dbhub.toml` changes + +### 8.4 Long-Term + +- Evaluate whether DuckDB's `ATTACH 'postgresql://...'` federation can replace the PostgreSQL source, simplifying to one connection +- Monitor Bifrost v1.6.0 release notes for tool-call handling improvements + +--- + +## 9. Critical Warnings for the Team + +1. **Never `git pull origin main` on the `duckdb-connector` branch without rebasing.** The 10+ type-union edits will conflict. Rebase, re-add `"duckdb"`, rebuild. + +2. **Never `git checkout main` and `pnpm build`.** The frontend build fails without `cd frontend && pnpm install`. Use `pnpm exec tsup` only. + +3. **Never skip `--allowed-hosts host.docker.internal`.** DBHub returns 403 for Docker requests, and the failure is silent on Bifrost's side — you'll see `Tools: 0` with no error. + +4. **Never embed `keys` in the Bifrost `/api/providers` payload.** Since v1.5.0, it is ignored silently. Use the separate `/api/providers/{name}/keys` endpoint. + +5. **Never use `localhost` in any Docker → Windows host URL.** Always `host.docker.internal`. + +--- + +## 10. Reference — Bifrost Auth Token + +The Bifrost admin credentials are: + +- Username: `DigiBull` +- Password: `DigiBull@2026` + +The Bearer token is `base64("DigiBull:DigiBull@2026")`: + +``` +RGlnaUJ1bGw6RGlnaUJ1bGxAMjAyNg== +``` + +Use this in the `Authorization: Bearer ` header for all Bifrost API calls. **Do not commit this token to any repo.** + +--- + +## 11. Contact / Ownership + +- Fork owner: `solutionsDigibull` (GitHub) +- Bifrost admin: `DigiBull` (local instance) +- Upstream DBHub: `github.com/bytebase/dbhub` +- Bifrost docs: `docs.getbifrost.ai` + +--- + +**Summary:** The pipeline is functional. All four DBHub tools (`execute_sql_postgres_logic`, `search_objects_postgres_logic`, `execute_sql_duckdb_local`, `search_objects_duckdb_local`) are discoverable and callable through Bifrost. Local LLM (`qwen2.5-coder:7b`) can generate correct tool calls. The only remaining ergonomic issue is that the model emits tool calls as text rather than native objects — workable today, improvable with a model swap. \ No newline at end of file diff --git a/dbhub.toml b/dbhub.toml new file mode 100644 index 00000000..99e952ef --- /dev/null +++ b/dbhub.toml @@ -0,0 +1,12 @@ +# dbhub.toml +[[sources]] +id = "postgres_logic" +description = "Production PostgreSQL database" +dsn = "postgresql://postgres:DigiBull@192.168.1.18:5432/logic_db_2?sslmode=disable" + +[[sources]] +id = "duckdb_local" +description = "Local DuckDB analytical database" +# Use an absolute path or a path relative to where you run DBHub. +# For an in-memory database, use: dsn = "duckdb://:memory:" +dsn = "duckdb:///C:/SRIM-DB-SETUP/Databases/Bom_storage.duckdb" \ No newline at end of file From 3ac555938b89505d550df86a4bd220aa47b522a4 Mon Sep 17 00:00:00 2001 From: "solutions@Digibull" Date: Wed, 30 Sep 2026 13:52:13 +0530 Subject: [PATCH 3/3] Refactor DBHub MCP tool-call pipeline script Updated PowerShell script for DBHub MCP tool-call pipeline, including changes to authorization headers, model usage, and JSON parsing for tool results. --- DBHub_MCP_TXTtoSQL.md | 134 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 111 insertions(+), 23 deletions(-) diff --git a/DBHub_MCP_TXTtoSQL.md b/DBHub_MCP_TXTtoSQL.md index fa61106e..b1629ff6 100644 --- a/DBHub_MCP_TXTtoSQL.md +++ b/DBHub_MCP_TXTtoSQL.md @@ -258,47 +258,135 @@ Expected: `ollama/qwen2.5-coder:7b`, `ollama/qwen3:8b`, etc. Due to `qwen2.5-coder:7b` not emitting native `tool_calls`, use this **4-step manual pattern**: ```powershell +# ============================================================ +# Bifrost + Ollama + DBHub MCP — end-to-end tool-call pipeline +# ============================================================ + $headers = @{ - "Authorization" = "Bearer " + "Authorization" = "Bearer RGlnaUJ1bGw6RGlnaUJ1bGxAMjAyNg==" "Content-Type" = "application/json" - "x-bf-mcp-include-clients" = "DBHub_MCP" # scopes tools to DBHub only + "x-bf-mcp-include-clients" = "DBHub_MCP" } +$BaseUrl = "http://192.168.1.18:8010" +$Model = "ollama/qwen2.5-coder:7b" +$CallId = "call_auto_1" +$UserMsg = "Show me 10 rows from gerber_checklist" + +# ------------------------------------------------------------ # 1. Ask the model -$chat = Invoke-RestMethod -Uri "http://localhost:8010/v1/chat/completions" ` +# ------------------------------------------------------------ +Write-Host "`n=== [1] Asking the model ===" -ForegroundColor Cyan + +$chat = Invoke-RestMethod -Uri "$BaseUrl/v1/chat/completions" ` -Method Post -Headers $headers -Body (@{ - model = "ollama/qwen2.5-coder:7b" - messages = @( @{ role = "user"; content = "Show me 3 rows from gerber_checklist" } ) + model = $Model + messages = @( @{ role = "user"; content = $UserMsg } ) } | ConvertTo-Json -Depth 5) -# 2. Parse the model's JSON output (emitted as content, not tool_calls) -$call = $chat.choices[0].message.content | ConvertFrom-Json +$raw = [string]$chat.choices[0].message.content -# 3. Execute through Bifrost -$result = Invoke-RestMethod -Uri "http://localhost:8010/v1/mcp/tool/execute" ` - -Method Post -Headers $headers -Body (@{ - id = "call_auto_1" +# ------------------------------------------------------------ +# 2. Parse the tool call (defensive) +# ------------------------------------------------------------ +Write-Host "`n=== [2] Parsing tool call ===" -ForegroundColor Cyan + +$raw = $raw.Trim() +$raw = $raw -replace '^\s*```(?:json)?\s*', '' -replace '\s*```\s*$', '' +if ($raw.StartsWith('{') -and -not $raw.EndsWith('}')) { $raw += '}' } + +try { $call = $raw | ConvertFrom-Json } catch { throw "Bad JSON:`n$raw" } +if (-not $call.name) { throw "No 'name' field:`n$raw" } + +$toolName = $call.name +$toolArgs = $call.arguments +Write-Host "Tool name : $toolName" +Write-Host "Tool args : $($toolArgs | ConvertTo-Json -Compress)" + +# ------------------------------------------------------------ +# 3. Execute through Bifrost (OpenAI function-call envelope) +# ------------------------------------------------------------ +Write-Host "`n=== [3] Executing tool via Bifrost ===" -ForegroundColor Cyan + +$execBody = @{ + id = $CallId type = "function" function = @{ - name = $call.name - arguments = ($call.arguments | ConvertTo-Json -Compress) + name = $toolName + arguments = ($toolArgs | ConvertTo-Json -Compress) } - } | ConvertTo-Json -Depth 5) +} | ConvertTo-Json -Depth 10 -Compress + +$result = Invoke-RestMethod -Uri "$BaseUrl/v1/mcp/tool/execute" ` + -Method Post -Headers $headers -Body $execBody + +# ------------------------------------------------------------ +# 3b. Extract the tool payload — FIXED (block form, not if/elseif expr) +# Bifrost returns: { role, content: "", tool_call_id } +# We parse the inner JSON and pull just the rows for a cleaner prompt. +# ------------------------------------------------------------ +if ($result -is [string]) { + $toolJsonStr = $result +} elseif ($null -ne $result.content) { + $toolJsonStr = if ($result.content -is [string]) { $result.content } else { $result.content | ConvertTo-Json -Depth 10 -Compress } +} else { + $toolJsonStr = $result | ConvertTo-Json -Depth 10 -Compress +} + +# Try to unwrap the DBHub envelope: { success, data: { statements: [ { rows: [...] } ] } } +$rowsPayload = $null +try { + $inner = $toolJsonStr | ConvertFrom-Json + if ($inner.data.statements[0].rows) { + $rowsPayload = $inner.data.statements[0].rows + } +} catch { } + +if ($rowsPayload) { + # Compact rows-only payload for the model + $toolContent = @{ + rows = $rowsPayload + count = $rowsPayload.Count + } | ConvertTo-Json -Depth 10 -Compress + Write-Host "Extracted $($rowsPayload.Count) rows for the model." -ForegroundColor Green +} else { + # Fall back to raw tool string + $toolContent = $toolJsonStr + Write-Host "Could not unwrap rows; passing raw tool output." -ForegroundColor Yellow +} + +# ------------------------------------------------------------ +# 4. Feed result back — with a system prompt so it summarizes +# instead of echoing raw JSON. +# ------------------------------------------------------------ +Write-Host "`n=== [4] Asking model to summarize tool result ===" -ForegroundColor Cyan + +$systemPrompt = @" +You are a helpful assistant. You will be given tool results as JSON. +Answer the user's question in plain, natural English. +Do NOT output raw JSON. Do NOT repeat the tool result verbatim. +Summarize the key fields and values. +"@ -# 4. Feed result back to model for natural-language answer (optional) -$answer = Invoke-RestMethod -Uri "http://localhost:8010/v1/chat/completions" ` +$answer = Invoke-RestMethod -Uri "$BaseUrl/v1/chat/completions" ` -Method Post -Headers $headers -Body (@{ - model = "ollama/qwen2.5-coder:7b" + model = $Model messages = @( - @{ role = "user"; content = "Show me 3 rows from gerber_checklist" } - @{ role = "assistant"; content = $null; tool_calls = @(@{ - id = "call_auto_1"; type = "function" - function = @{ name = $call.name; arguments = ($call.arguments | ConvertTo-Json -Compress) } + @{ role = "system"; content = $systemPrompt } + @{ role = "user"; content = $UserMsg } + @{ role = "assistant"; content = ""; tool_calls = @(@{ + id = $CallId + type = "function" + function = @{ + name = $toolName + arguments = ($toolArgs | ConvertTo-Json -Compress) + } })} - @{ role = "tool"; tool_call_id = "call_auto_1"; content = $result.content } + @{ role = "tool"; tool_call_id = $CallId; content = $toolContent } ) } | ConvertTo-Json -Depth 10) +Write-Host "`n=== Final answer ===" -ForegroundColor Yellow $answer.choices[0].message.content ``` @@ -392,4 +480,4 @@ Use this in the `Authorization: Bearer ` header for all Bifrost API calls --- -**Summary:** The pipeline is functional. All four DBHub tools (`execute_sql_postgres_logic`, `search_objects_postgres_logic`, `execute_sql_duckdb_local`, `search_objects_duckdb_local`) are discoverable and callable through Bifrost. Local LLM (`qwen2.5-coder:7b`) can generate correct tool calls. The only remaining ergonomic issue is that the model emits tool calls as text rather than native objects — workable today, improvable with a model swap. \ No newline at end of file +**Summary:** The pipeline is functional. All four DBHub tools (`execute_sql_postgres_logic`, `search_objects_postgres_logic`, `execute_sql_duckdb_local`, `search_objects_duckdb_local`) are discoverable and callable through Bifrost. Local LLM (`qwen2.5-coder:7b`) can generate correct tool calls. The only remaining ergonomic issue is that the model emits tool calls as text rather than native objects — workable today, improvable with a model swap.