perf: defer Databricks Connect initialization to background thread

.cursor/.gitignore

@@ -0,0 +1 @@
+plans/

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to the Databricks Notebook Studio extension will be document
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.6] - 2026-04-14
+
+### Changed
+- **Databricks Connect startup performance**: Kernel now reports ready in ~1-2 seconds instead of ~15 seconds by deferring Spark session initialization to a background thread
+  - A `LazySparkSession` proxy is placed in the namespace immediately so `spark` is available right away
+  - The proxy transparently blocks on first use (e.g. `spark.sql()`) until the real session is connected
+  - Pure Python cells can execute immediately without waiting for Databricks Connect
+  - Background thread sends a `spark_ready` status update to VS Code when initialization completes
+  - Thread-safe stdout writes prevent message interleaving between main and background threads
+
 ## [0.4.5] - 2026-04-02
 
 ### Fixed

package.json

@@ -2,7 +2,7 @@
   "name": "databricks-notebook-studio",
   "displayName": "Databricks Notebook Studio",
   "description": "Visualize Databricks .py notebooks with rich DataFrame display, interactive tables, column sorting/resizing, and multi-profile authentication",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "license": "MIT",
   "type": "commonjs",
   "publisher": "databricks-notebook-studio",
@@ -144,8 +144,8 @@
         },
         "databricks-notebook.kernelStartupTimeout": {
           "type": "number",
-          "default": 15000,
-          "description": "Timeout for kernel startup in milliseconds. Databricks Connect initialization can take 15-25 seconds on first run."
+          "default": 30000,
+          "description": "Timeout for kernel startup in milliseconds. With deferred Spark initialization, the kernel typically starts in 1-2 seconds."
         },
         "databricks-notebook.enableScrollableOutput": {
           "type": "boolean",
@@ -231,7 +231,7 @@
     "lint:fix": "eslint src --ext ts --fix",
     "test": "node ./out/test/runTest.js",
     "test:unit": "tsc --declaration false --declarationMap false && mocha 'out/test/{parser,profileManager,codeTransform,constants,serializer,requestCache,catalogService,tabManager,cellOperations,notebookDiagnosticProvider,persistentExecutor,kernelManager,pythonKernelController,kernelControls,pythonCompletion,linting,localImports,dotenv}.test.js'",
-    "test:python": "cd src/python && python3 test_kernel_runner.py",
+    "test:python": "cd src/python && python3 test_kernel_runner.py && python3 test_spark_auth.py",
     "test:coverage": "nyc --reporter=lcov --reporter=text npm run test:unit",
     "ci": "npm run type-check && npm run lint && npm run test:unit && npm run test:python && npm run compile"
   },

src/kernels/persistentExecutor.ts

@@ -29,6 +29,7 @@ import {
   logKernelStatusDebug,
   createReadyWaiter,
   getKernelStartupTimeout,
+  showSparkReadyNotification,
 } from './utils';
 
 /**
@@ -340,6 +341,12 @@ export class PersistentExecutor implements vscode.Disposable {
         return;
       }
 
+      // Handle deferred spark initialization completion
+      if (message.type === 'spark_ready') {
+        this.handleSparkReadySignal(message as KernelResponse);
+        return;
+      }
+
       // Handle widget input requests from Python
       if (message.type === 'input_request') {
         this.handleWidgetInputRequest(message as WidgetInputRequest);
@@ -376,6 +383,22 @@ export class PersistentExecutor implements vscode.Disposable {
     showKernelStatusNotifications(statusInfo);
   }
 
+  /**
+   * Handle deferred spark initialization completion.
+   * Sent by background thread in kernel_runner.py after DatabricksSession is connected.
+   */
+  private handleSparkReadySignal(response: KernelResponse): void {
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    const sparkStatus = (response as { spark_status?: string }).spark_status;
+    if (sparkStatus) {
+      this._sparkStatus = sparkStatus;
+      if (this._debugMode) {
+        console.debug(`[Executor] Spark ready: ${sparkStatus}`);
+      }
+      showSparkReadyNotification(sparkStatus);
+    }
+  }
+
   /**
    * Handle pending request response
    */

src/kernels/utils.ts

@@ -219,12 +219,27 @@ export function showKernelStatusNotifications(statusInfo: KernelStatusInfo): voi
   }
 
   // Notify about spark status
+  // Skip "INITIALIZING:" status - the real notification comes via spark_ready message
   if (statusInfo.sparkStatus) {
     if (statusInfo.sparkStatus.startsWith('OK:')) {
       showInfoMessage(`Kernel: ${statusInfo.sparkStatus}`);
     } else if (statusInfo.sparkStatus.startsWith('WARN:')) {
       showWarningMessage(`Kernel: ${statusInfo.sparkStatus}`);
     }
+    // INITIALIZING: is silently logged (notification comes when spark_ready arrives)
+  }
+}
+
+/**
+ * Show notification when deferred spark initialization completes.
+ * Called when the background spark init thread finishes and sends spark_ready.
+ * @param sparkStatus - Final spark status string
+ */
+export function showSparkReadyNotification(sparkStatus: string): void {
+  if (sparkStatus.startsWith('OK:')) {
+    showInfoMessage(`Kernel: ${sparkStatus}`);
+  } else if (sparkStatus.startsWith('WARN:')) {
+    showWarningMessage(`Kernel: ${sparkStatus}`);
   }
 }