Upgrade databricks-sql-kernel to 1.0.0 (#938)

* chore: upgrade databricks-sql-kernel to 1.0.0

Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>

* docs: remove stale kernel pin references

Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>

---------

Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>

Author: vuanhphung

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Release History
 
 # Unreleased
+- Upgraded the optional `databricks-sql-kernel` backend dependency to 1.0.0.
 - Kernel metadata filters no longer collapse empty strings to `None`; empty patterns therefore match nothing. Existing `%`/`*` catalog wildcard handling is unchanged (PECOBLR-4221).
 - Kernel backend (`use_kernel=True`): OAuth **M2M with a JWT private-key client assertion** (RFC 7523) is now supported. Pass `oauth_client_id` + `oauth_jwt_key_file` + `oauth_jwt_kid` (with optional `oauth_jwt_passphrase` for an encrypted PKCS#8 key, `oauth_jwt_algorithm` defaulting to `RS256`, `oauth_scopes`, and `token_url` for the IdP token endpoint) and the connector routes them to the kernel's `auth_type="oauth-m2m-jwt"`, which signs a short-lived assertion with the private key instead of sending a client secret. The kernel owns the token lifecycle. A private-key file is treated as unambiguous JWT M2M intent and is mutually exclusive with `oauth_client_secret` / `credentials_provider` (both raise `NotSupportedError`). Verified end-to-end against an Azure Databricks workspace with the service principal's public certificate registered on its Entra ID app registration. Requires `databricks-sql-kernel >= 0.2.0` with JWT support.
 - Kernel backend (`use_kernel=True`): OAuth U2M with `auth_type="databricks-oauth"` now forwards the connector's `databricks-sql-python` OAuth-app bundle (`client_id` + `sql offline_access` scopes + redirect port) into the kernel, so a bare U2M connection authenticates as `databricks-sql-python` — parity with the Thrift path — instead of inheriting the kernel's own `databricks-sql-connector` default. A caller-supplied `oauth_client_id` (with its coupled `oauth_redirect_port`) is honored, as is a caller-supplied `oauth_scopes`; absent one, the connector default (`sql offline_access`) is forwarded. Note: the kernel binds a single U2M redirect port, so unlike the Thrift path (which tries the full `8020..8024` range) the kernel path uses only one port and does not fall back to the next port if it is already bound — pass `oauth_redirect_port` (with `oauth_client_id`) to pick a free one on a port collision (PECOBLR-4040)

poetry.lock

@@ -48,37 +48,28 @@ pybreaker = "^1.0.0"
 requests-kerberos = {version = "^0.15.0", optional = true}
 # Optional Rust kernel backend for ``use_kernel=True`` (PyO3 wheel).
 # Pulled in only via the ``[kernel]`` extra below. The published wheel
-# is ``abi3`` with ``Requires-Python: >=3.10`` (built ``abi3-py310``),
-# so the dependency is gated to Python >= 3.10: on 3.8/3.9 the
-# ``[kernel]`` extra resolves to nothing and ``use_kernel=True`` raises
-# a clear ImportError at runtime (see backend/kernel/_errors.py).
+# is ``abi3`` with ``Requires-Python: >=3.9`` (built ``abi3-py39``); the
+# dependency remains gated to the connector's Python >= 3.10 support floor.
 #
-# Floor is 0.2.0 (``^0.2.0`` == ``>=0.2.0,<0.3.0``). The kernel is
-# pre-1.0, so each 0.x minor may carry breaking changes — the ``<0.3.0``
-# cap means we bump this deliberately when the kernel ships 0.3.0 rather
-# than letting a potentially-breaking minor flow in automatically. 0.2.0
-# keeps the same Requires-Python (>=3.10) and pyarrow (>=23.0.1,<24) pin
-# as 0.1.x, so the gating below is unchanged.
-databricks-sql-kernel = {version = "^0.2.0", optional = true, python = ">=3.10"}
+# Floor is 1.0.0 (``^1.0.0`` == ``>=1.0.0,<2.0.0``), allowing compatible
+# 1.x releases while keeping the next major upgrade deliberate. The kernel's
+# pyarrow constraint remains >=23.0.1,<24.
+databricks-sql-kernel = {version = "^1.0.0", optional = true, python = ">=3.10"}
 
 
 [tool.poetry.extras]
 pyarrow = ["pyarrow"]
 # ``pip install databricks-sql-connector[kernel]`` adds the Rust kernel
-# backend so ``use_kernel=True`` works. No-op on Python < 3.10 (the
-# wheel's floor) — those users get a runtime ImportError if they pass
-# ``use_kernel=True``.
+# backend so ``use_kernel=True`` works.
 #
 # The kernel result path (``backend/kernel/result_set.py``) needs
 # pyarrow, but it is NOT listed in this extra on purpose: the published
 # kernel wheel declares ``pyarrow>=23.0.1,<24`` as a hard runtime
 # dependency, so ``pip install ...[kernel]`` already pulls a compatible
 # pyarrow transitively. Listing bare ``pyarrow`` here additionally
 # forces poetry to co-resolve an unconstrained pyarrow against the
-# kernel's ``>=23.0.1,<24`` (which itself requires Python >=3.10) across
-# the connector's full 3.8–3.14 support matrix, which is unsatisfiable
-# on 3.8/3.9 and breaks ``poetry lock``. The kernel's own dependency
-# metadata is the single source of truth for the pyarrow floor.
+# kernel's ``>=23.0.1,<24``. The kernel's own dependency metadata is the
+# single source of truth for the pyarrow floor.
 kernel = ["databricks-sql-kernel"]
 
 [tool.poetry.group.dev.dependencies]

pyproject.toml

@@ -154,19 +154,10 @@ def _kernel_session_accepts_kwarg(name: str) -> bool:
 
     The kernel ``Session`` is a PyO3 class with a **fixed** signature (no
     ``**kwargs`` catch-all), so forwarding a kwarg it doesn't declare raises
-    ``TypeError`` at construction. The phase-7 identity/telemetry kwargs
-    (``driver_name`` etc.) only exist on wheels newer than the pinned
-    ``^0.2.0`` (whose ``Session`` accepts none of them), so we must gate them
-    on what the actually-installed wheel supports rather than pass them
-    unconditionally. Falls **closed** (returns ``False``) when the signature
-    can't be introspected: a PyO3 class only exposes ``__text_signature__``
-    (and thus an introspectable signature) when built with
-    ``#[pyo3(signature=...)]``; otherwise ``inspect.signature`` raises
-    ``ValueError``. Since the pinned ``^0.2.0`` ``Session`` accepts none of
-    these kwargs, forwarding one it doesn't declare is a hard ``TypeError`` at
-    construction that breaks every ``use_kernel=True`` connection, whereas
-    omitting one the wheel *would* have accepted only loses telemetry
-    richness — so we omit the kwarg on introspection failure.
+    ``TypeError`` at construction, so we gate kwargs on what the installed
+    wheel supports. Falls **closed** (returns ``False``) when the signature
+    can't be introspected because omitting an accepted telemetry kwarg is safer
+    than forwarding an unsupported one.
     """
     try:
         params = inspect.signature(_kernel.Session).parameters
@@ -181,9 +172,7 @@ def _kernel_telemetry_kwargs(options: Dict[str, Any]) -> Dict[str, Any]:
     """Build phase-7 telemetry/system kwargs for ``databricks_sql_kernel.Session``.
 
     Only kwargs the installed ``Session`` constructor actually accepts are
-    returned; on the pinned ``^0.2.0`` wheel (which predates phase 7) this is
-    empty, so ``open_session`` doesn't break with ``TypeError`` on a wheel
-    that doesn't yet know these kwargs.
+    returned, preventing ``TypeError`` when the binding lacks an option.
     """
     system = TelemetryHelper.get_driver_system_configuration()
     candidates: Dict[str, Any] = {