From b6ee424899a7c7a0cd160890c2d1deb9fe5ab607 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 30 Sep 2026 19:48:58 -0400 Subject: [PATCH 01/11] perf(sqlalchemy-bigquery): parallelize compliance tests with worker-partitioned datasets and pytest-xdist --- ci/get_package_shards.py | 9 +-- packages/sqlalchemy-bigquery/noxfile.py | 35 +++++++---- .../sqlalchemy_bigquery/provision.py | 62 +++++++++++++++++++ .../sqlalchemy_dialect_compliance/conftest.py | 29 ++++++++- .../test_dialect_compliance.py | 11 +++- 5 files changed, 121 insertions(+), 25 deletions(-) create mode 100644 packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index d0dd4e7d4ea2..558abd4a99be 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -49,14 +49,7 @@ } # Packages temporarily excluded from CI test execution. -# NOTE: 'sqlalchemy-bigquery' is temporarily excluded to allow testing in this PR -# to complete due to an upstream packaging issue in sqlalchemy (duplicate normalized -# extra name 'mssql-pymssql' under strict uv PEP 621 parsing in sqlalchemy==2.1.0rc2, -# pulled via global UV_PRERELEASE=allow). Awaiting team feedback on a long-term -# solution (e.g. package migration out of the monorepo or adjusting workflow settings). -EXCLUDED_PACKAGES = { - "sqlalchemy-bigquery", -} +EXCLUDED_PACKAGES = set() def get_package_directories(): diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index f9ebdeb8b331..bdadc468387c 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -170,6 +170,11 @@ def wrapper(*args, **kwargs): nox.options.stop_on_first_error = True nox.options.error_on_missing_interpreters = True +# NOTE: venv_backend="virtualenv" is used to bypass an upstream packaging issue +# in sqlalchemy (duplicate normalized extra name 'mssql-pymssql' under strict uv +# PEP 621 parsing in sqlalchemy==2.1.0rc2, pulled via global UV_PRERELEASE=allow). +VENV_BACKEND = "virtualenv" + @nox.session(python=DEFAULT_PYTHON_VERSION) @_calculate_duration @@ -289,7 +294,7 @@ def install_unittest_dependencies(session, *constraints): session.install("-e", ".", *constraints) -@nox.session(python=ALL_PYTHON) +@nox.session(python=ALL_PYTHON, venv_backend=VENV_BACKEND) @nox.parametrize( "protobuf_implementation", ["python", "upb"], @@ -407,6 +412,7 @@ def _run_system_test_logic(session, test_type): "mock", "pytest", "pytest-rerunfailures", + "pytest-xdist", "google-cloud-testutils", "-c", constraints_path, @@ -425,12 +431,19 @@ def _run_system_test_logic(session, test_type): # Execution logic if test_type == "compliance": + num_workers = os.environ.get("COMPLIANCE_WORKERS", "4") + xdist_args = [] + if not any(arg.startswith("-n") for arg in session.posargs): + if num_workers not in ("0", "1"): + xdist_args = [f"-n={num_workers}", "--dist=loadscope"] + session.run( "py.test", "-vv", + *xdist_args, f"--junitxml=compliance_{session.python}_sponge_log.xml", - "--reruns=3", - "--reruns-delay=60", + "--reruns=2", + "--reruns-delay=30", "--only-rerun=Exceeded rate limits", "--only-rerun=Already Exists", "--only-rerun=Not found", @@ -450,7 +463,7 @@ def _run_system_test_logic(session, test_type): ) -@nox.session(python="3.12") +@nox.session(python="3.12", venv_backend=VENV_BACKEND) @nox.parametrize("test_type", ["system", "system_noextras", "compliance"]) @_calculate_duration def system(session, test_type): @@ -458,21 +471,21 @@ def system(session, test_type): _run_system_test_logic(session, test_type) -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS, venv_backend=VENV_BACKEND) @_calculate_duration def system_noextras(session): """Run the system test suite without extras.""" _run_system_test_logic(session, "system_noextras") -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) +@nox.session(python=DEFAULT_PYTHON_VERSION, venv_backend=VENV_BACKEND) @_calculate_duration def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" _run_system_test_logic(session, "compliance") -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python=DEFAULT_PYTHON_VERSION, venv_backend=VENV_BACKEND) @_calculate_duration def cover(session): """Run the final coverage report. @@ -486,7 +499,7 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python="3.10") +@nox.session(python="3.10", venv_backend=VENV_BACKEND) @_calculate_duration def docs(session): """Build the docs for this library.""" @@ -524,7 +537,7 @@ def docs(session): ) -@nox.session(python="3.10") +@nox.session(python="3.10", venv_backend=VENV_BACKEND) @_calculate_duration def docfx(session): """Build the docfx yaml files for this library.""" @@ -573,7 +586,7 @@ def docfx(session): ) -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python=DEFAULT_PYTHON_VERSION, venv_backend=VENV_BACKEND) @nox.parametrize( "protobuf_implementation", ["python", "upb"], @@ -697,7 +710,7 @@ def mypy(session): session.skip("mypy tests are not yet supported") -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python=DEFAULT_PYTHON_VERSION, venv_backend=VENV_BACKEND) @nox.parametrize( "protobuf_implementation", ["python", "upb"], diff --git a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py new file mode 100644 index 000000000000..c1b04a66e1d7 --- /dev/null +++ b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py @@ -0,0 +1,62 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import os + +import google.cloud.bigquery +import test_utils.prefixer +from sqlalchemy.engine import make_url +from sqlalchemy.testing.provision import ( + create_db, + drop_db, + follower_url_from_main, +) + +prefixer = test_utils.prefixer.Prefixer( + "python-bigquery-sqlalchemy", "tests/compliance" +) + + +def _dataset_id_from_ident(ident: str) -> str: + """Derive a deterministic BigQuery dataset ID for an xdist follower ident.""" + run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") + if not run_prefix: + run_prefix = prefixer.create_prefix() + return f"{run_prefix}_{ident}" + + +@follower_url_from_main.for_db("bigquery") +def _bigquery_follower_url_from_main(url, ident): + url = make_url(url) + dataset_id = _dataset_id_from_ident(ident) + return url.set(database=dataset_id) + + +@create_db.for_db("bigquery") +def _bigquery_create_db(cfg, eng, ident): + dataset_id = _dataset_id_from_ident(ident) + with contextlib.closing(google.cloud.bigquery.Client()) as client: + dataset_ref = google.cloud.bigquery.DatasetReference(client.project, dataset_id) + dataset = google.cloud.bigquery.Dataset(dataset_ref) + # Set 1-hour expiration as safety net in case of process termination + dataset.default_table_expiration_ms = 3600 * 1000 + client.create_dataset(dataset, exists_ok=True) + + +@drop_db.for_db("bigquery") +def _bigquery_drop_db(cfg, eng, ident): + dataset_id = _dataset_id_from_ident(ident) + with contextlib.closing(google.cloud.bigquery.Client()) as client: + client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True) diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py index c5d69ced03dd..40838e5f937f 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py @@ -18,6 +18,7 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import contextlib +import os import re import traceback @@ -33,6 +34,7 @@ ) import sqlalchemy_bigquery.base +import sqlalchemy_bigquery.provision # noqa: F401 sqlalchemy_bigquery.BigQueryDialect.preexecute_autoincrement_sequences = True @@ -70,14 +72,35 @@ def visit_delete(self, delete_stmt, *args, **kw): def pytest_sessionstart(session): - dataset_id = prefixer.create_prefix() - session.config.option.dburi = [f"bigquery:///{dataset_id}"] + if hasattr(session.config, "workerinput"): + # In a pytest-xdist worker process: + # Dataset creation and URL binding are dynamically provisioned + # per-worker by sqlalchemy_bigquery.provision hooks. + _pytest_sessionstart(session) + return + + # Master process (or single-process sequential run): + run_prefix = prefixer.create_prefix() + os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix + master_dataset_id = f"{run_prefix}_master" + session.config.option.dburi = [f"bigquery:///{master_dataset_id}"] with contextlib.closing(google.cloud.bigquery.Client()) as client: - client.create_dataset(dataset_id) + dataset_ref = google.cloud.bigquery.DatasetReference( + client.project, master_dataset_id + ) + dataset = google.cloud.bigquery.Dataset(dataset_ref) + dataset.default_table_expiration_ms = 3600 * 1000 + client.create_dataset(dataset, exists_ok=True) _pytest_sessionstart(session) def pytest_sessionfinish(session): + if hasattr(session.config, "workerinput"): + # Worker dataset teardown is handled by provision.drop_follower_db + # on the controller node. + _pytest_sessionfinish(session) + return + dataset_id = config.db.dialect.dataset_id _pytest_sessionfinish(session) with contextlib.closing(google.cloud.bigquery.Client()) as client: diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py index dde343c62429..bca5d6e39f27 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py @@ -644,6 +644,11 @@ def test_no_results_for_non_returning_insert(cls): PostCompileParamsTest ) # BQ adds backticks to bind parameters, causing failure of tests TODO: fix this? del QuotedNameArgumentTest # Quotes aren't allowed in BigQuery table names. -del ( - WindowFunctionTest.test_window_rows_between -) # test expects BQ to return sorted results +# Test expects BQ to return sorted results (renamed to _w_caching in SQLAlchemy 2.1+) +for _test_name in ("test_window_rows_between", "test_window_rows_between_w_caching"): + if hasattr(WindowFunctionTest, _test_name): + delattr(WindowFunctionTest, _test_name) + +# BigQuery requires dataset qualification for views, which TableViaSelectTest does not provide. +if "TableViaSelectTest" in globals(): + del TableViaSelectTest From 50ba897ecace1f3a8cce2ca33c6953e825b2fdf2 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 30 Sep 2026 20:03:09 -0400 Subject: [PATCH 02/11] fix(sqlalchemy-bigquery): synchronize worker dataset prefix across xdist processes Initialize COMPLIANCE_RUN_PREFIX in pytest_configure before workers are spawned and propagate it via workerinput and environment variables. Also cache fallback prefix in provision.py and recognize --numprocesses in noxfile. --- packages/sqlalchemy-bigquery/noxfile.py | 5 ++- .../sqlalchemy_bigquery/provision.py | 1 + .../sqlalchemy_dialect_compliance/conftest.py | 35 ++++++++++++++++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index bdadc468387c..85a0314590ee 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -433,7 +433,10 @@ def _run_system_test_logic(session, test_type): if test_type == "compliance": num_workers = os.environ.get("COMPLIANCE_WORKERS", "4") xdist_args = [] - if not any(arg.startswith("-n") for arg in session.posargs): + if not any( + arg.startswith("-n") or arg.startswith("--numprocesses") + for arg in session.posargs + ): if num_workers not in ("0", "1"): xdist_args = [f"-n={num_workers}", "--dist=loadscope"] diff --git a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py index c1b04a66e1d7..d10690e50f1e 100644 --- a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py +++ b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py @@ -34,6 +34,7 @@ def _dataset_id_from_ident(ident: str) -> str: run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") if not run_prefix: run_prefix = prefixer.create_prefix() + os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix return f"{run_prefix}_{ident}" diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py index 40838e5f937f..137de6575518 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py @@ -26,6 +26,9 @@ import test_utils.prefixer from sqlalchemy.testing import config from sqlalchemy.testing.plugin.pytestplugin import * # noqa +from sqlalchemy.testing.plugin.pytestplugin import ( + pytest_configure as _pytest_configure, +) from sqlalchemy.testing.plugin.pytestplugin import ( pytest_sessionfinish as _pytest_sessionfinish, ) @@ -71,6 +74,21 @@ def visit_delete(self, delete_stmt, *args, **kw): sqlalchemy_bigquery.base.BigQueryCompiler.visit_delete = visit_delete +def pytest_configure(config): + if hasattr(config, "workerinput"): + prefix = config.workerinput.get("compliance_run_prefix") + if prefix: + os.environ["COMPLIANCE_RUN_PREFIX"] = prefix + else: + if "COMPLIANCE_RUN_PREFIX" not in os.environ: + os.environ["COMPLIANCE_RUN_PREFIX"] = prefixer.create_prefix() + _pytest_configure(config) + + +def pytest_configure_node(node): + node.workerinput["compliance_run_prefix"] = os.environ.get("COMPLIANCE_RUN_PREFIX") + + def pytest_sessionstart(session): if hasattr(session.config, "workerinput"): # In a pytest-xdist worker process: @@ -80,8 +98,10 @@ def pytest_sessionstart(session): return # Master process (or single-process sequential run): - run_prefix = prefixer.create_prefix() - os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix + run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") + if not run_prefix: + run_prefix = prefixer.create_prefix() + os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix master_dataset_id = f"{run_prefix}_master" session.config.option.dburi = [f"bigquery:///{master_dataset_id}"] with contextlib.closing(google.cloud.bigquery.Client()) as client: @@ -101,10 +121,17 @@ def pytest_sessionfinish(session): _pytest_sessionfinish(session) return - dataset_id = config.db.dialect.dataset_id _pytest_sessionfinish(session) + run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") with contextlib.closing(google.cloud.bigquery.Client()) as client: - client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True) + if hasattr(config, "db") and config.db is not None: + dataset_id = config.db.dialect.dataset_id + client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True) + elif run_prefix: + client.delete_dataset( + f"{run_prefix}_master", delete_contents=True, not_found_ok=True + ) + for dataset in client.list_datasets(): if prefixer.should_cleanup(dataset.dataset_id): client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) From a4e8eabecda181e298250c90d5a9b5b491758289 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 04:35:44 -0400 Subject: [PATCH 03/11] fix(sqlalchemy-bigquery): support generate_driver_url and bind worker dburi in xdist sessionstart Implement generate_driver_url provision hook for bigquery dialect so generate_db_urls resolves properly. Set dburi on workers and ensure worker datasets exist during worker sessionstart to prevent NoSectionError. --- .../sqlalchemy_bigquery/provision.py | 13 +++++++++ .../sqlalchemy_dialect_compliance/conftest.py | 27 ++++++++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py index d10690e50f1e..0f2588f91df4 100644 --- a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py +++ b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py @@ -22,6 +22,7 @@ create_db, drop_db, follower_url_from_main, + generate_driver_url, ) prefixer = test_utils.prefixer.Prefixer( @@ -38,6 +39,18 @@ def _dataset_id_from_ident(ident: str) -> str: return f"{run_prefix}_{ident}" +@generate_driver_url.for_db("bigquery") +def _bigquery_generate_driver_url(url, driver, query_str): + url = make_url(url) + if driver and driver != "bigquery": + new_url = url.set(drivername=f"bigquery+{driver}") + else: + new_url = url.set(drivername="bigquery") + if query_str: + new_url = new_url.update_query_string(query_str) + return new_url + + @follower_url_from_main.for_db("bigquery") def _bigquery_follower_url_from_main(url, ident): url = make_url(url) diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py index 137de6575518..4d62d4b8c325 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py @@ -82,6 +82,10 @@ def pytest_configure(config): else: if "COMPLIANCE_RUN_PREFIX" not in os.environ: os.environ["COMPLIANCE_RUN_PREFIX"] = prefixer.create_prefix() + + run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") + master_dataset_id = f"{run_prefix}_master" + config.option.dburi = [f"bigquery:///{master_dataset_id}"] _pytest_configure(config) @@ -90,18 +94,29 @@ def pytest_configure_node(node): def pytest_sessionstart(session): + run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") + if not run_prefix: + run_prefix = prefixer.create_prefix() + os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix + if hasattr(session.config, "workerinput"): # In a pytest-xdist worker process: - # Dataset creation and URL binding are dynamically provisioned - # per-worker by sqlalchemy_bigquery.provision hooks. + # Each worker connects to its own partition ({run_prefix}_{follower_ident}). + # Ensure the worker's dataset exists and bind dburi to worker partition. + ident = session.config.workerinput.get("follower_ident") + worker_dataset_id = f"{run_prefix}_{ident}" if ident else f"{run_prefix}_worker" + session.config.option.dburi = [f"bigquery:///{worker_dataset_id}"] + with contextlib.closing(google.cloud.bigquery.Client()) as client: + dataset_ref = google.cloud.bigquery.DatasetReference( + client.project, worker_dataset_id + ) + dataset = google.cloud.bigquery.Dataset(dataset_ref) + dataset.default_table_expiration_ms = 3600 * 1000 + client.create_dataset(dataset, exists_ok=True) _pytest_sessionstart(session) return # Master process (or single-process sequential run): - run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") - if not run_prefix: - run_prefix = prefixer.create_prefix() - os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix master_dataset_id = f"{run_prefix}_master" session.config.option.dburi = [f"bigquery:///{master_dataset_id}"] with contextlib.closing(google.cloud.bigquery.Client()) as client: From f0c1f3b9bb89e670c790de961ab814d64ba2f27d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 05:59:09 -0400 Subject: [PATCH 04/11] feat(sqlalchemy-bigquery): add DOUBLE compilation, docs fallback, and unit tests for provision.py --- packages/sqlalchemy-bigquery/docs/conf.py | 16 ++- .../sqlalchemy_bigquery/base.py | 2 +- .../sqlalchemy_bigquery/provision.py | 20 +++- .../tests/unit/test_compiler.py | 37 +++++++ .../tests/unit/test_provision.py | 103 ++++++++++++++++++ 5 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 packages/sqlalchemy-bigquery/tests/unit/test_provision.py diff --git a/packages/sqlalchemy-bigquery/docs/conf.py b/packages/sqlalchemy-bigquery/docs/conf.py index 2a3478cc1616..b9b09445a11f 100644 --- a/packages/sqlalchemy-bigquery/docs/conf.py +++ b/packages/sqlalchemy-bigquery/docs/conf.py @@ -359,7 +359,6 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "python": ("https://python.readthedocs.org/en/latest/", None), "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ( "https://googleapis.dev/python/google-api-core/latest/", @@ -370,6 +369,21 @@ "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), } +# Check reachability of the Python standard library inventory before attaching it. +# Because Sphinx is run with `-W` (warnings as errors) in CI, an external network +# failure or upstream outage on docs.python.org would otherwise treat the missing +# inventory as a fatal error and fail the build. +try: + import urllib.request + + with urllib.request.urlopen("https://docs.python.org/3/objects.inv", timeout=2): + intersphinx_mapping["python"] = ( + "https://python.readthedocs.org/en/latest/", + None, + ) +except Exception: + pass + # Napoleon settings napoleon_google_docstring = True diff --git a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/base.py b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/base.py index e2922890cea0..db11393d97ae 100644 --- a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/base.py +++ b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/base.py @@ -600,7 +600,7 @@ def visit_BOOLEAN(self, type_, **kw): def visit_FLOAT(self, type_, **kw): return "FLOAT64" - visit_REAL = visit_FLOAT + visit_REAL = visit_DOUBLE = visit_DOUBLE_PRECISION = visit_FLOAT def visit_STRING(self, type_, **kw): if (type_.length is not None) and isinstance( diff --git a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py index 0f2588f91df4..043ad27c1525 100644 --- a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py +++ b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py @@ -13,10 +13,11 @@ # limitations under the License. import contextlib +import datetime import os +import uuid import google.cloud.bigquery -import test_utils.prefixer from sqlalchemy.engine import make_url from sqlalchemy.testing.provision import ( create_db, @@ -25,16 +26,25 @@ generate_driver_url, ) -prefixer = test_utils.prefixer.Prefixer( - "python-bigquery-sqlalchemy", "tests/compliance" -) +try: + import test_utils.prefixer # pragma: NO COVER + + prefixer = test_utils.prefixer.Prefixer( # pragma: NO COVER + "python-bigquery-sqlalchemy", "tests/compliance" + ) +except ImportError: + prefixer = None def _dataset_id_from_ident(ident: str) -> str: """Derive a deterministic BigQuery dataset ID for an xdist follower ident.""" run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") if not run_prefix: - run_prefix = prefixer.create_prefix() + if prefixer: + run_prefix = prefixer.create_prefix() + else: + now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d%H%M%S") + run_prefix = f"python_bigquery_sqlalchemy_tests_compliance_{now}_{uuid.uuid4().hex[:6]}" os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix return f"{run_prefix}_{ident}" diff --git a/packages/sqlalchemy-bigquery/tests/unit/test_compiler.py b/packages/sqlalchemy-bigquery/tests/unit/test_compiler.py index 77f55dd7bf80..4fae78e7b1e6 100644 --- a/packages/sqlalchemy-bigquery/tests/unit/test_compiler.py +++ b/packages/sqlalchemy-bigquery/tests/unit/test_compiler.py @@ -450,3 +450,40 @@ def compile_custom_intersect(element, compiler, **kwargs): found_sql = q.compile(faux_conn).string assert found_sql == expected_sql + + +def test_type_compiler_double_methods(): + from sqlalchemy_bigquery.base import BigQueryTypeCompiler + + assert BigQueryTypeCompiler.visit_DOUBLE == BigQueryTypeCompiler.visit_FLOAT + assert ( + BigQueryTypeCompiler.visit_DOUBLE_PRECISION == BigQueryTypeCompiler.visit_FLOAT + ) + + +_double_types = [ + getattr(sqlalchemy, name) + for name in ("DOUBLE", "DOUBLE_PRECISION", "Double") + if hasattr(sqlalchemy, name) +] + + +@pytest.mark.skipif( + not _double_types, + reason="DOUBLE types not present in this SQLAlchemy version", +) +@pytest.mark.parametrize("type_cls", _double_types or [None]) +def test_double_types_compile_to_float64(type_cls): + import sqlalchemy_bigquery + + dialect = sqlalchemy_bigquery.BigQueryDialect() + assert dialect.type_compiler.process(type_cls()) == "FLOAT64" + + +def test_float_literal_pyformat_bindparam(): + import sqlalchemy_bigquery + + dialect = sqlalchemy_bigquery.BigQueryDialect(paramstyle="pyformat") + stmt = sqlalchemy.select(sqlalchemy.literal(15.7563)) + compiled = stmt.compile(dialect=dialect) + assert "%(param_1:FLOAT64)s" in str(compiled) diff --git a/packages/sqlalchemy-bigquery/tests/unit/test_provision.py b/packages/sqlalchemy-bigquery/tests/unit/test_provision.py new file mode 100644 index 000000000000..9109eb7554fa --- /dev/null +++ b/packages/sqlalchemy-bigquery/tests/unit/test_provision.py @@ -0,0 +1,103 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest import mock + +import pytest +from sqlalchemy.engine import make_url + +from sqlalchemy_bigquery import provision + + +def test_dataset_id_from_ident_with_existing_env(monkeypatch): + monkeypatch.setenv("COMPLIANCE_RUN_PREFIX", "custom_prefix") + dataset_id = provision._dataset_id_from_ident("gw0") + assert dataset_id == "custom_prefix_gw0" + + +def test_dataset_id_from_ident_without_env(monkeypatch): + monkeypatch.delenv("COMPLIANCE_RUN_PREFIX", raising=False) + monkeypatch.setattr(provision, "prefixer", None) + dataset_id = provision._dataset_id_from_ident("gw1") + cached_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") + assert cached_prefix is not None + assert dataset_id == f"{cached_prefix}_gw1" + + +def test_dataset_id_from_ident_with_prefixer(monkeypatch): + monkeypatch.delenv("COMPLIANCE_RUN_PREFIX", raising=False) + mock_prefixer = mock.Mock() + mock_prefixer.create_prefix.return_value = "mock_pfx" + monkeypatch.setattr(provision, "prefixer", mock_prefixer) + dataset_id = provision._dataset_id_from_ident("gw5") + assert dataset_id == "mock_pfx_gw5" + assert os.environ.get("COMPLIANCE_RUN_PREFIX") == "mock_pfx" + + +@pytest.mark.parametrize( + "driver, query_str, expected_drivername, expected_query", + [ + (None, None, "bigquery", {}), + ("bigquery", None, "bigquery", {}), + ("bigquery", "param=1", "bigquery", {"param": "1"}), + ("custom", None, "bigquery+custom", {}), + ("custom", "param=2", "bigquery+custom", {"param": "2"}), + ], +) +def test_generate_driver_url(driver, query_str, expected_drivername, expected_query): + base_url = "bigquery:///test_master" + result = provision._bigquery_generate_driver_url(base_url, driver, query_str) + assert result.drivername == expected_drivername + if expected_query: + for k, v in expected_query.items(): + assert result.query.get(k) == v + + +def test_follower_url_from_main(monkeypatch): + monkeypatch.setenv("COMPLIANCE_RUN_PREFIX", "run_123") + base_url = make_url("bigquery:///master_dataset") + follower_url = provision._bigquery_follower_url_from_main(base_url, "gw2") + assert follower_url.database == "run_123_gw2" + + +def test_create_db(monkeypatch): + monkeypatch.setenv("COMPLIANCE_RUN_PREFIX", "run_456") + mock_client = mock.MagicMock() + mock_client.project = "test-project" + mock_cfg = mock.Mock() + mock_cfg.db.url = make_url("bigquery:///test_dataset") + + with mock.patch("google.cloud.bigquery.Client", return_value=mock_client): + provision._bigquery_create_db(mock_cfg, mock.Mock(), "gw3") + + mock_client.create_dataset.assert_called_once() + created_dataset = mock_client.create_dataset.call_args[0][0] + assert created_dataset.dataset_id == "run_456_gw3" + assert created_dataset.default_table_expiration_ms == 3600 * 1000 + assert mock_client.create_dataset.call_args[1].get("exists_ok") is True + + +def test_drop_db(monkeypatch): + monkeypatch.setenv("COMPLIANCE_RUN_PREFIX", "run_789") + mock_client = mock.MagicMock() + mock_cfg = mock.Mock() + mock_cfg.db.url = make_url("bigquery:///test_dataset") + + with mock.patch("google.cloud.bigquery.Client", return_value=mock_client): + provision._bigquery_drop_db(mock_cfg, mock.Mock(), "gw4") + + mock_client.delete_dataset.assert_called_once_with( + "run_789_gw4", delete_contents=True, not_found_ok=True + ) From 9b3a5fa578cb9b15bcaad4fa817b546a2afa7fd4 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 07:09:21 -0400 Subject: [PATCH 05/11] style(docs): narrow exception handling for intersphinx fallback --- packages/sqlalchemy-bigquery/docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/sqlalchemy-bigquery/docs/conf.py b/packages/sqlalchemy-bigquery/docs/conf.py index b9b09445a11f..4da3f9a32770 100644 --- a/packages/sqlalchemy-bigquery/docs/conf.py +++ b/packages/sqlalchemy-bigquery/docs/conf.py @@ -374,6 +374,7 @@ # failure or upstream outage on docs.python.org would otherwise treat the missing # inventory as a fatal error and fail the build. try: + import urllib.error import urllib.request with urllib.request.urlopen("https://docs.python.org/3/objects.inv", timeout=2): @@ -381,7 +382,7 @@ "https://python.readthedocs.org/en/latest/", None, ) -except Exception: +except (urllib.error.URLError, TimeoutError, OSError): pass From 82e4027908f703372a84c9e12827d2c564623256 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 07:28:34 -0400 Subject: [PATCH 06/11] refactor(sqlalchemy-bigquery): deduplicate dataset provisioning and streamline conftest hooks --- .../sqlalchemy_dialect_compliance/conftest.py | 66 ++++++++----------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py index 4d62d4b8c325..ce54fe5ebf08 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py @@ -74,6 +74,24 @@ def visit_delete(self, delete_stmt, *args, **kw): sqlalchemy_bigquery.base.BigQueryCompiler.visit_delete = visit_delete +def _ensure_dataset(dataset_id: str) -> None: + """Ensure a compliance test dataset exists with a 1-hour expiration safety net.""" + with contextlib.closing(google.cloud.bigquery.Client()) as client: + dataset_ref = google.cloud.bigquery.DatasetReference(client.project, dataset_id) + dataset = google.cloud.bigquery.Dataset(dataset_ref) + dataset.default_table_expiration_ms = 3600 * 1000 + client.create_dataset(dataset, exists_ok=True) + + +def _resolve_dataset_id(cfg) -> str: + """Resolve the dataset ID for the current runner process (worker or master).""" + run_prefix = os.environ["COMPLIANCE_RUN_PREFIX"] + if hasattr(cfg, "workerinput"): + ident = cfg.workerinput.get("follower_ident") + return f"{run_prefix}_{ident}" if ident else f"{run_prefix}_worker" + return f"{run_prefix}_master" + + def pytest_configure(config): if hasattr(config, "workerinput"): prefix = config.workerinput.get("compliance_run_prefix") @@ -83,9 +101,8 @@ def pytest_configure(config): if "COMPLIANCE_RUN_PREFIX" not in os.environ: os.environ["COMPLIANCE_RUN_PREFIX"] = prefixer.create_prefix() - run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") - master_dataset_id = f"{run_prefix}_master" - config.option.dburi = [f"bigquery:///{master_dataset_id}"] + dataset_id = _resolve_dataset_id(config) + config.option.dburi = [f"bigquery:///{dataset_id}"] _pytest_configure(config) @@ -94,38 +111,9 @@ def pytest_configure_node(node): def pytest_sessionstart(session): - run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") - if not run_prefix: - run_prefix = prefixer.create_prefix() - os.environ["COMPLIANCE_RUN_PREFIX"] = run_prefix - - if hasattr(session.config, "workerinput"): - # In a pytest-xdist worker process: - # Each worker connects to its own partition ({run_prefix}_{follower_ident}). - # Ensure the worker's dataset exists and bind dburi to worker partition. - ident = session.config.workerinput.get("follower_ident") - worker_dataset_id = f"{run_prefix}_{ident}" if ident else f"{run_prefix}_worker" - session.config.option.dburi = [f"bigquery:///{worker_dataset_id}"] - with contextlib.closing(google.cloud.bigquery.Client()) as client: - dataset_ref = google.cloud.bigquery.DatasetReference( - client.project, worker_dataset_id - ) - dataset = google.cloud.bigquery.Dataset(dataset_ref) - dataset.default_table_expiration_ms = 3600 * 1000 - client.create_dataset(dataset, exists_ok=True) - _pytest_sessionstart(session) - return - - # Master process (or single-process sequential run): - master_dataset_id = f"{run_prefix}_master" - session.config.option.dburi = [f"bigquery:///{master_dataset_id}"] - with contextlib.closing(google.cloud.bigquery.Client()) as client: - dataset_ref = google.cloud.bigquery.DatasetReference( - client.project, master_dataset_id - ) - dataset = google.cloud.bigquery.Dataset(dataset_ref) - dataset.default_table_expiration_ms = 3600 * 1000 - client.create_dataset(dataset, exists_ok=True) + dataset_id = _resolve_dataset_id(session.config) + session.config.option.dburi = [f"bigquery:///{dataset_id}"] + _ensure_dataset(dataset_id) _pytest_sessionstart(session) @@ -139,9 +127,11 @@ def pytest_sessionfinish(session): _pytest_sessionfinish(session) run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") with contextlib.closing(google.cloud.bigquery.Client()) as client: - if hasattr(config, "db") and config.db is not None: - dataset_id = config.db.dialect.dataset_id - client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True) + db = getattr(session.config, "db", None) or getattr(config, "db", None) + if db is not None and hasattr(db.dialect, "dataset_id"): + client.delete_dataset( + db.dialect.dataset_id, delete_contents=True, not_found_ok=True + ) elif run_prefix: client.delete_dataset( f"{run_prefix}_master", delete_contents=True, not_found_ok=True From 0ab98442c8c73c8d28ec113f8248c6deb8d2286a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 11:44:57 -0400 Subject: [PATCH 07/11] refactor(sqlalchemy-bigquery): share dataset lifecycle helpers between provision and conftest --- .../sqlalchemy_bigquery/provision.py | 21 ++++++++---- .../sqlalchemy_dialect_compliance/conftest.py | 34 +++++++------------ .../tests/unit/test_provision.py | 25 ++++++++++++++ 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py index 043ad27c1525..4ed4b1801985 100644 --- a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py +++ b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/provision.py @@ -68,19 +68,28 @@ def _bigquery_follower_url_from_main(url, ident): return url.set(database=dataset_id) -@create_db.for_db("bigquery") -def _bigquery_create_db(cfg, eng, ident): - dataset_id = _dataset_id_from_ident(ident) +def ensure_dataset(dataset_id: str) -> None: + """Ensure a BigQuery dataset exists with a 1-hour expiration safety net.""" with contextlib.closing(google.cloud.bigquery.Client()) as client: dataset_ref = google.cloud.bigquery.DatasetReference(client.project, dataset_id) dataset = google.cloud.bigquery.Dataset(dataset_ref) - # Set 1-hour expiration as safety net in case of process termination dataset.default_table_expiration_ms = 3600 * 1000 client.create_dataset(dataset, exists_ok=True) +def drop_dataset(dataset_id: str) -> None: + """Drop a BigQuery dataset and its contents if it exists.""" + with contextlib.closing(google.cloud.bigquery.Client()) as client: + client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True) + + +@create_db.for_db("bigquery") +def _bigquery_create_db(cfg, eng, ident): + dataset_id = _dataset_id_from_ident(ident) + ensure_dataset(dataset_id) + + @drop_db.for_db("bigquery") def _bigquery_drop_db(cfg, eng, ident): dataset_id = _dataset_id_from_ident(ident) - with contextlib.closing(google.cloud.bigquery.Client()) as client: - client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True) + drop_dataset(dataset_id) diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py index ce54fe5ebf08..3e1a6d23e14f 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py @@ -37,6 +37,9 @@ ) import sqlalchemy_bigquery.base + +# flake8: F401 is ignored because importing this module registers the BigQuery dialect +# provisioning hooks (@create_db, @drop_db, etc.) with sqlalchemy.testing.provision. import sqlalchemy_bigquery.provision # noqa: F401 sqlalchemy_bigquery.BigQueryDialect.preexecute_autoincrement_sequences = True @@ -74,21 +77,14 @@ def visit_delete(self, delete_stmt, *args, **kw): sqlalchemy_bigquery.base.BigQueryCompiler.visit_delete = visit_delete -def _ensure_dataset(dataset_id: str) -> None: - """Ensure a compliance test dataset exists with a 1-hour expiration safety net.""" - with contextlib.closing(google.cloud.bigquery.Client()) as client: - dataset_ref = google.cloud.bigquery.DatasetReference(client.project, dataset_id) - dataset = google.cloud.bigquery.Dataset(dataset_ref) - dataset.default_table_expiration_ms = 3600 * 1000 - client.create_dataset(dataset, exists_ok=True) - - def _resolve_dataset_id(cfg) -> str: """Resolve the dataset ID for the current runner process (worker or master).""" run_prefix = os.environ["COMPLIANCE_RUN_PREFIX"] if hasattr(cfg, "workerinput"): ident = cfg.workerinput.get("follower_ident") - return f"{run_prefix}_{ident}" if ident else f"{run_prefix}_worker" + if ident: + return sqlalchemy_bigquery.provision._dataset_id_from_ident(ident) + return f"{run_prefix}_worker" return f"{run_prefix}_master" @@ -113,7 +109,7 @@ def pytest_configure_node(node): def pytest_sessionstart(session): dataset_id = _resolve_dataset_id(session.config) session.config.option.dburi = [f"bigquery:///{dataset_id}"] - _ensure_dataset(dataset_id) + sqlalchemy_bigquery.provision.ensure_dataset(dataset_id) _pytest_sessionstart(session) @@ -126,17 +122,13 @@ def pytest_sessionfinish(session): _pytest_sessionfinish(session) run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") - with contextlib.closing(google.cloud.bigquery.Client()) as client: - db = getattr(session.config, "db", None) or getattr(config, "db", None) - if db is not None and hasattr(db.dialect, "dataset_id"): - client.delete_dataset( - db.dialect.dataset_id, delete_contents=True, not_found_ok=True - ) - elif run_prefix: - client.delete_dataset( - f"{run_prefix}_master", delete_contents=True, not_found_ok=True - ) + db = getattr(session.config, "db", None) or getattr(config, "db", None) + if db is not None and hasattr(db.dialect, "dataset_id"): + sqlalchemy_bigquery.provision.drop_dataset(db.dialect.dataset_id) + elif run_prefix: + sqlalchemy_bigquery.provision.drop_dataset(f"{run_prefix}_master") + with contextlib.closing(google.cloud.bigquery.Client()) as client: for dataset in client.list_datasets(): if prefixer.should_cleanup(dataset.dataset_id): client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) diff --git a/packages/sqlalchemy-bigquery/tests/unit/test_provision.py b/packages/sqlalchemy-bigquery/tests/unit/test_provision.py index 9109eb7554fa..6004f7e541f2 100644 --- a/packages/sqlalchemy-bigquery/tests/unit/test_provision.py +++ b/packages/sqlalchemy-bigquery/tests/unit/test_provision.py @@ -101,3 +101,28 @@ def test_drop_db(monkeypatch): mock_client.delete_dataset.assert_called_once_with( "run_789_gw4", delete_contents=True, not_found_ok=True ) + + +def test_ensure_dataset(): + mock_client = mock.MagicMock() + mock_client.project = "test-project" + + with mock.patch("google.cloud.bigquery.Client", return_value=mock_client): + provision.ensure_dataset("custom_dataset_123") + + mock_client.create_dataset.assert_called_once() + created = mock_client.create_dataset.call_args[0][0] + assert created.dataset_id == "custom_dataset_123" + assert created.default_table_expiration_ms == 3600 * 1000 + assert mock_client.create_dataset.call_args[1].get("exists_ok") is True + + +def test_drop_dataset(): + mock_client = mock.MagicMock() + + with mock.patch("google.cloud.bigquery.Client", return_value=mock_client): + provision.drop_dataset("custom_dataset_456") + + mock_client.delete_dataset.assert_called_once_with( + "custom_dataset_456", delete_contents=True, not_found_ok=True + ) From 796b5ef635c685b47df7c1fc164a3c71006ee24d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 12:09:12 -0400 Subject: [PATCH 08/11] refactor(sqlalchemy-bigquery): simplify pytestplugin imports and hook calls in conftest --- .../sqlalchemy_dialect_compliance/conftest.py | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py index 3e1a6d23e14f..8ead9373f63a 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py @@ -25,22 +25,14 @@ import google.cloud.bigquery.dbapi.connection import test_utils.prefixer from sqlalchemy.testing import config -from sqlalchemy.testing.plugin.pytestplugin import * # noqa -from sqlalchemy.testing.plugin.pytestplugin import ( - pytest_configure as _pytest_configure, -) -from sqlalchemy.testing.plugin.pytestplugin import ( - pytest_sessionfinish as _pytest_sessionfinish, -) -from sqlalchemy.testing.plugin.pytestplugin import ( - pytest_sessionstart as _pytest_sessionstart, -) +from sqlalchemy.testing.plugin import pytestplugin -import sqlalchemy_bigquery.base +# flake8: F401, F403 are ignored because SQLAlchemy requires importing its testing +# plugin fixtures and hooks directly into the pytest conftest namespace. +from sqlalchemy.testing.plugin.pytestplugin import * # noqa: F401, F403 -# flake8: F401 is ignored because importing this module registers the BigQuery dialect -# provisioning hooks (@create_db, @drop_db, etc.) with sqlalchemy.testing.provision. -import sqlalchemy_bigquery.provision # noqa: F401 +import sqlalchemy_bigquery.base +import sqlalchemy_bigquery.provision sqlalchemy_bigquery.BigQueryDialect.preexecute_autoincrement_sequences = True @@ -99,7 +91,7 @@ def pytest_configure(config): dataset_id = _resolve_dataset_id(config) config.option.dburi = [f"bigquery:///{dataset_id}"] - _pytest_configure(config) + pytestplugin.pytest_configure(config) def pytest_configure_node(node): @@ -110,17 +102,17 @@ def pytest_sessionstart(session): dataset_id = _resolve_dataset_id(session.config) session.config.option.dburi = [f"bigquery:///{dataset_id}"] sqlalchemy_bigquery.provision.ensure_dataset(dataset_id) - _pytest_sessionstart(session) + pytestplugin.pytest_sessionstart(session) def pytest_sessionfinish(session): if hasattr(session.config, "workerinput"): # Worker dataset teardown is handled by provision.drop_follower_db # on the controller node. - _pytest_sessionfinish(session) + pytestplugin.pytest_sessionfinish(session) return - _pytest_sessionfinish(session) + pytestplugin.pytest_sessionfinish(session) run_prefix = os.environ.get("COMPLIANCE_RUN_PREFIX") db = getattr(session.config, "db", None) or getattr(config, "db", None) if db is not None and hasattr(db.dialect, "dataset_id"): From afcad643741da1cd7663c12679e2165eb499605a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 12:13:34 -0400 Subject: [PATCH 09/11] docs(sqlalchemy-bigquery): add docstrings to compliance conftest hook functions --- .../tests/sqlalchemy_dialect_compliance/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py index 8ead9373f63a..f19672222f91 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/conftest.py @@ -54,6 +54,7 @@ def visit_delete(self, delete_stmt, *args, **kw): + """Compile DELETE statements, appending WHERE true during test teardown.""" text = super(sqlalchemy_bigquery.base.BigQueryCompiler, self).visit_delete( delete_stmt, *args, **kw ) @@ -81,6 +82,7 @@ def _resolve_dataset_id(cfg) -> str: def pytest_configure(config): + """Configure pytest session, establishing compliance run prefix and target dburi.""" if hasattr(config, "workerinput"): prefix = config.workerinput.get("compliance_run_prefix") if prefix: @@ -95,10 +97,12 @@ def pytest_configure(config): def pytest_configure_node(node): + """Propagate the compliance run prefix from the controller to worker nodes.""" node.workerinput["compliance_run_prefix"] = os.environ.get("COMPLIANCE_RUN_PREFIX") def pytest_sessionstart(session): + """Ensure the target dataset exists before running compliance tests in the session.""" dataset_id = _resolve_dataset_id(session.config) session.config.option.dburi = [f"bigquery:///{dataset_id}"] sqlalchemy_bigquery.provision.ensure_dataset(dataset_id) @@ -106,6 +110,7 @@ def pytest_sessionstart(session): def pytest_sessionfinish(session): + """Tear down datasets and clean up leaked test datasets after session completion.""" if hasattr(session.config, "workerinput"): # Worker dataset teardown is handled by provision.drop_follower_db # on the controller node. From f2c9feaf21071e9f2d149d97a101c59365d54833 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 12:14:55 -0400 Subject: [PATCH 10/11] docs(sqlalchemy-bigquery): clarify sorted results comment in compliance suite --- .../sqlalchemy_dialect_compliance/test_dialect_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py index bca5d6e39f27..b449de2d9aec 100644 --- a/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py +++ b/packages/sqlalchemy-bigquery/tests/sqlalchemy_dialect_compliance/test_dialect_compliance.py @@ -644,7 +644,7 @@ def test_no_results_for_non_returning_insert(cls): PostCompileParamsTest ) # BQ adds backticks to bind parameters, causing failure of tests TODO: fix this? del QuotedNameArgumentTest # Quotes aren't allowed in BigQuery table names. -# Test expects BQ to return sorted results (renamed to _w_caching in SQLAlchemy 2.1+) +# Test expects BQ to return sorted results, which it does not do. (renamed to _w_caching in SQLAlchemy 2.1+) for _test_name in ("test_window_rows_between", "test_window_rows_between_w_caching"): if hasattr(WindowFunctionTest, _test_name): delattr(WindowFunctionTest, _test_name) From 8ac2d669eabd8299b23f106a87d9201cf97a9e24 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 1 Oct 2026 12:39:20 -0400 Subject: [PATCH 11/11] refactor(sqlalchemy-bigquery): simplify compliance test xdist worker configuration --- packages/sqlalchemy-bigquery/noxfile.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 85a0314590ee..c99032072a4a 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -431,14 +431,9 @@ def _run_system_test_logic(session, test_type): # Execution logic if test_type == "compliance": - num_workers = os.environ.get("COMPLIANCE_WORKERS", "4") - xdist_args = [] - if not any( - arg.startswith("-n") or arg.startswith("--numprocesses") - for arg in session.posargs - ): - if num_workers not in ("0", "1"): - xdist_args = [f"-n={num_workers}", "--dist=loadscope"] + xdist_args = ["-n=4", "--dist=loadscope"] + if any(arg.startswith(("-n", "--numprocesses")) for arg in session.posargs): + xdist_args = [] session.run( "py.test",