From e9a015e86411ae6b775db6bddf76523c7401f063 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 15:47:28 +0200 Subject: [PATCH 01/15] Add pgdevkit.testdb naming (slugify, branch, workspace db name) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_015bpLodYTShC96vqY6gqsQy --- pgdevkit/testdb/__init__.py | 0 pgdevkit/testdb/constants.py | 9 +++++++ pgdevkit/testdb/naming.py | 41 +++++++++++++++++++++++++++++ pyproject.toml | 2 ++ tests/testdb/__init__.py | 0 tests/testdb/test_naming.py | 51 ++++++++++++++++++++++++++++++++++++ 6 files changed, 103 insertions(+) create mode 100644 pgdevkit/testdb/__init__.py create mode 100644 pgdevkit/testdb/constants.py create mode 100644 pgdevkit/testdb/naming.py create mode 100644 tests/testdb/__init__.py create mode 100644 tests/testdb/test_naming.py diff --git a/pgdevkit/testdb/__init__.py b/pgdevkit/testdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pgdevkit/testdb/constants.py b/pgdevkit/testdb/constants.py new file mode 100644 index 0000000..09f37a6 --- /dev/null +++ b/pgdevkit/testdb/constants.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +CONTAINER_NAME = "pgdevkit-postgres" +IMAGE = "pgvector/pgvector:pg18-trixie" +HOST = "localhost" +PORT = 54322 +USER = "postgres" +PASSWORD = "testpwd" +PG_SPEED_FLAGS = ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] diff --git a/pgdevkit/testdb/naming.py b/pgdevkit/testdb/naming.py new file mode 100644 index 0000000..7a21240 --- /dev/null +++ b/pgdevkit/testdb/naming.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import hashlib +import re +import subprocess +from pathlib import Path + +_INVALID_CHARS = re.compile(r"[^a-z0-9_]+") +_MAX_SLUG_LEN = 30 + + +def slugify(value: str) -> str: + """Lowercase, replace invalid chars with '_', truncate+hash if too long.""" + slug = _INVALID_CHARS.sub("_", value.lower()).strip("_") + if not slug: + slug = "x" + if len(slug) <= _MAX_SLUG_LEN: + return slug + digest = hashlib.sha256(slug.encode()).hexdigest()[:8] + return f"{slug[:_MAX_SLUG_LEN]}_{digest}" + + +def current_branch(cwd: Path | None = None) -> str: + """Return the branch checked out in the git worktree rooted at cwd.""" + result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def workspace_db_name(project_name: str, branch: str) -> str: + """Compute a Postgres-safe, collision-resistant database name for this + project+branch. A second slugify pass over the joined string guarantees + the result stays under Postgres's 63-byte identifier limit even when + both inputs are already at the per-component truncation limit.""" + joined = f"{slugify(project_name)}_{slugify(branch)}" + return slugify(joined) diff --git a/pyproject.toml b/pyproject.toml index 574c049..4a1ec68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,11 +31,13 @@ test = [ "docker>=7.1.0", "psycopg[binary]>=3.2.0", "pytest>=9.1.0", + "pytest-asyncio>=0.24.0", "pytest-env>=1.1.0", ] [tool.pytest.ini_options] pythonpath = ["."] +asyncio_mode = "auto" [tool.pytest_env] PGDB_TEST_POSTGRES_HOST = "localhost" diff --git a/tests/testdb/__init__.py b/tests/testdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/testdb/test_naming.py b/tests/testdb/test_naming.py new file mode 100644 index 0000000..f394bf4 --- /dev/null +++ b/tests/testdb/test_naming.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +from pgdevkit.testdb.naming import current_branch, slugify, workspace_db_name + + +def test_slugify_lowercases_and_replaces_invalid_chars(): + assert slugify("MDMApp") == "mdmapp" + assert slugify("feature/customer-contacts") == "feature_customer_contacts" + + +def test_slugify_strips_leading_trailing_underscores(): + assert slugify("--hello--") == "hello" + + +def test_slugify_truncates_long_input_and_appends_hash(): + long_name = "a" * 50 + result = slugify(long_name) + assert len(result) == 30 + 1 + 8 + assert result.startswith("a" * 30 + "_") + + +def test_slugify_is_deterministic(): + long_name = "worktree-procrastinate-job-events-extra-long-branch-name" + assert slugify(long_name) == slugify(long_name) + + +def test_workspace_db_name_differs_by_branch(): + a = workspace_db_name("mdmapp", "main") + b = workspace_db_name("mdmapp", "multi_lng") + assert a != b + assert a == "mdmapp_main" + + +def test_workspace_db_name_stays_under_postgres_identifier_limit(): + name = workspace_db_name("a" * 50, "b" * 50) + assert len(name) <= 63 + + +def test_current_branch_reads_the_checked_out_branch(tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "t@example.com"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "t"], cwd=tmp_path, check=True) + subprocess.run(["git", "checkout", "-q", "-b", "my-feature"], cwd=tmp_path, check=True) + (tmp_path / "f.txt").write_text("x", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=tmp_path, check=True) + + assert current_branch(tmp_path) == "my-feature" From 70f2c40cccc7d74aaed91c0733def59635656128 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 15:51:07 +0200 Subject: [PATCH 02/15] Add pgdevkit.testdb project config loading Implements ProjectConfig dataclass and load_config() function for reading project configuration from pyproject.toml [tool.pgdevkit] section, with sensible defaults for name, database_dir, env_prefix, extensions, and root. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_015bpLodYTShC96vqY6gqsQy --- pgdevkit/testdb/config.py | 49 ++++++++++++++++++++++++++++++++++ tests/testdb/test_config.py | 53 +++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 pgdevkit/testdb/config.py create mode 100644 tests/testdb/test_config.py diff --git a/pgdevkit/testdb/config.py b/pgdevkit/testdb/config.py new file mode 100644 index 0000000..814e956 --- /dev/null +++ b/pgdevkit/testdb/config.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +try: + import tomllib +except ImportError: # pragma: no cover - Python < 3.11 + import tomli as tomllib # type: ignore[no-redef] + + +@dataclass(frozen=True) +class ProjectConfig: + name: str + database_dir: str = "database" + env_prefix: str = "" + extensions: tuple[str, ...] = () + root: Path = field(default_factory=Path) + + def __post_init__(self) -> None: + if not self.env_prefix: + object.__setattr__(self, "env_prefix", f"{self.name.upper()}_") + + +def _find_pyproject(start: Path) -> Path | None: + for directory in [start, *start.parents]: + candidate = directory / "pyproject.toml" + if candidate.exists(): + return candidate + return None + + +def load_config(start: Path | None = None) -> ProjectConfig: + start = (start or Path.cwd()).resolve() + pyproject = _find_pyproject(start) + root = pyproject.parent if pyproject else start + + section: dict = {} + if pyproject is not None: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + section = data.get("tool", {}).get("pgdevkit", {}) + + return ProjectConfig( + name=section.get("name") or root.name, + database_dir=section.get("database_dir", "database"), + env_prefix=section.get("env_prefix", ""), + extensions=tuple(section.get("extensions", [])), + root=root, + ) diff --git a/tests/testdb/test_config.py b/tests/testdb/test_config.py new file mode 100644 index 0000000..b865fde --- /dev/null +++ b/tests/testdb/test_config.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from pathlib import Path + +from pgdevkit.testdb.config import load_config + + +def test_defaults_when_no_pyproject(tmp_path: Path): + project = tmp_path / "myproj" + project.mkdir() + config = load_config(project) + assert config.name == "myproj" + assert config.database_dir == "database" + assert config.env_prefix == "MYPROJ_" + assert config.extensions == () + assert config.root == project + + +def test_reads_tool_pgdevkit_section(tmp_path: Path): + (tmp_path / "pyproject.toml").write_text( + """ +[tool.pgdevkit] +name = "mdmapp" +database_dir = "db" +env_prefix = "MDM_" +extensions = ["vector"] +""", + encoding="utf-8", + ) + config = load_config(tmp_path) + assert config.name == "mdmapp" + assert config.database_dir == "db" + assert config.env_prefix == "MDM_" + assert config.extensions == ("vector",) + + +def test_env_prefix_defaults_from_name(tmp_path: Path): + (tmp_path / "pyproject.toml").write_text( + '[tool.pgdevkit]\nname = "ccmt"\n', encoding="utf-8" + ) + config = load_config(tmp_path) + assert config.env_prefix == "CCMT_" + + +def test_searches_upward_for_pyproject(tmp_path: Path): + (tmp_path / "pyproject.toml").write_text( + '[tool.pgdevkit]\nname = "root_project"\n', encoding="utf-8" + ) + subdir = tmp_path / "src" / "nested" + subdir.mkdir(parents=True) + config = load_config(subdir) + assert config.name == "root_project" + assert config.root == tmp_path From 9bc025147a1b3306101da1fc10a523ee5ea6b32d Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 16:59:10 +0200 Subject: [PATCH 03/15] Add pgdevkit.testdb container lifecycle (podman) --- pgdevkit/testdb/container.py | 70 ++++++++++++++++++++++++++++++++++ tests/testdb/conftest.py | 9 +++++ tests/testdb/test_container.py | 35 +++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 pgdevkit/testdb/container.py create mode 100644 tests/testdb/conftest.py create mode 100644 tests/testdb/test_container.py diff --git a/pgdevkit/testdb/container.py b/pgdevkit/testdb/container.py new file mode 100644 index 0000000..f38a60e --- /dev/null +++ b/pgdevkit/testdb/container.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import subprocess +import time + +import psycopg + +from . import constants + + +def _podman(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["podman", *args], capture_output=True, text=True, check=check) + + +def _container_status() -> str | None: + """Return 'running', 'exited', etc., or None if the container doesn't exist.""" + result = _podman( + "inspect", constants.CONTAINER_NAME, "--format", "{{.State.Status}}", check=False + ) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def _create_container() -> None: + result = _podman( + "run", "-d", + "--name", constants.CONTAINER_NAME, + "-p", f"{constants.PORT}:5432", + "-e", f"POSTGRES_USER={constants.USER}", + "-e", f"POSTGRES_PASSWORD={constants.PASSWORD}", + constants.IMAGE, + "postgres", *constants.PG_SPEED_FLAGS, + check=False, + ) + if result.returncode != 0 and "already in use" in result.stderr: + _podman("start", constants.CONTAINER_NAME) + return + if result.returncode != 0: + raise RuntimeError(f"podman run failed: {result.stderr}") + + +def _wait_ready(timeout: float = 30.0) -> None: + deadline = time.monotonic() + timeout + dsn = ( + f"postgresql://{constants.USER}:{constants.PASSWORD}" + f"@{constants.HOST}:{constants.PORT}/postgres?connect_timeout=2" + ) + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + with psycopg.connect(dsn): + return + except Exception as e: # noqa: BLE001 + last_error = e + time.sleep(0.5) + raise RuntimeError(f"Postgres did not become ready within {timeout}s: {last_error}") + + +def ensure_container() -> None: + """Idempotently ensure the shared pgdevkit-postgres container is running + and accepting connections.""" + status = _container_status() + if status == "running": + return + if status is not None: + _podman("start", constants.CONTAINER_NAME) + else: + _create_container() + _wait_ready() diff --git a/tests/testdb/conftest.py b/tests/testdb/conftest.py new file mode 100644 index 0000000..23f9a24 --- /dev/null +++ b/tests/testdb/conftest.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import shutil + +import pytest + +requires_podman = pytest.mark.skipif( + shutil.which("podman") is None, reason="podman is not installed" +) diff --git a/tests/testdb/test_container.py b/tests/testdb/test_container.py new file mode 100644 index 0000000..e4b90a4 --- /dev/null +++ b/tests/testdb/test_container.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import psycopg + +from pgdevkit.testdb import constants +from pgdevkit.testdb.container import ensure_container +from tests.testdb.conftest import requires_podman + + +def _admin_dsn() -> str: + return ( + f"postgresql://{constants.USER}:{constants.PASSWORD}" + f"@{constants.HOST}:{constants.PORT}/postgres?connect_timeout=5" + ) + + +@requires_podman +def test_ensure_container_starts_and_accepts_connections(): + ensure_container() + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) + + +@requires_podman +def test_ensure_container_is_idempotent(): + ensure_container() + ensure_container() # must not raise, must not error on "name already in use" + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) From 61e60c383d92527771e625b93a9b4233959a5cf9 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 17:00:15 +0200 Subject: [PATCH 04/15] Update uv.lock for pytest-asyncio dependency (Task 1 follow-up) --- uv.lock | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/uv.lock b/uv.lock index 2e48f3c..52b2739 100644 --- a/uv.lock +++ b/uv.lock @@ -293,6 +293,7 @@ test = [ { name = "docker" }, { name = "psycopg", extra = ["binary"] }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-env" }, ] @@ -311,6 +312,7 @@ test = [ { name = "docker", specifier = ">=7.1.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, { name = "pytest", specifier = ">=9.1.0" }, + { name = "pytest-asyncio", specifier = ">=0.24.0" }, { name = "pytest-env", specifier = ">=1.1.0" }, ] @@ -406,6 +408,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + [[package]] name = "pytest-env" version = "1.6.0" From 65612c0551f7562b430162c738632a11e871230f Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 17:06:14 +0200 Subject: [PATCH 05/15] Add pgdevkit.testdb SQL execution helper Implement query.execute() to run SQL statements against a PostgreSQL database via async connection, handling multiple statements and returning rows from the final statement if it produced any. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code --- pgdevkit/testdb/query.py | 17 +++++++++++++ tests/testdb/test_query.py | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 pgdevkit/testdb/query.py create mode 100644 tests/testdb/test_query.py diff --git a/pgdevkit/testdb/query.py b/pgdevkit/testdb/query.py new file mode 100644 index 0000000..163fdc2 --- /dev/null +++ b/pgdevkit/testdb/query.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import psycopg +from psycopg.rows import dict_row + + +async def execute(dsn: str, sql: str) -> list[dict] | None: + """Run one or more ';'-separated statements against dsn. Returns the + rows of the final statement if it produced any, else None.""" + statements = [s.strip() for s in sql.split(";") if s.strip()] + last_rows: list[dict] | None = None + async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as con: + for stmt in statements: + async with con.cursor(row_factory=dict_row) as cur: + await cur.execute(stmt) + last_rows = await cur.fetchall() if cur.description else None + return last_rows diff --git a/tests/testdb/test_query.py b/tests/testdb/test_query.py new file mode 100644 index 0000000..9c6b3db --- /dev/null +++ b/tests/testdb/test_query.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import psycopg +import pytest + +from pgdevkit.testdb import constants, query +from pgdevkit.testdb.container import ensure_container +from tests.testdb.conftest import requires_podman + +TEST_DB = "pgdevkit_query_selftest" + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +def _db_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{TEST_DB}" + + +@pytest.fixture +def query_test_db(): + ensure_container() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + con.execute(f'CREATE DATABASE "{TEST_DB}"') + yield + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + + +@requires_podman +async def test_execute_returns_rows_for_select(query_test_db): + rows = await query.execute(_db_dsn(), "SELECT 1 AS one, 2 AS two") + assert rows == [{"one": 1, "two": 2}] + + +@requires_podman +async def test_execute_returns_none_for_ddl(query_test_db): + rows = await query.execute(_db_dsn(), "CREATE TABLE t (id int)") + assert rows is None + + +@requires_podman +async def test_execute_runs_multiple_statements_and_returns_last(query_test_db): + rows = await query.execute( + _db_dsn(), + "CREATE TABLE t2 (id int); INSERT INTO t2 VALUES (1); SELECT * FROM t2", + ) + assert rows == [{"id": 1}] From d4ed87d06d13dc9e9741e4791ec5426288981589 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 18:05:22 +0200 Subject: [PATCH 06/15] Add pgdevkit.testdb schema apply (database-in-source walker) Ports the database-in-source-layout SQL walker and .test_data.json seeding from the postgres-test-setup skill's start_postgres.py, parameterized by database_dir instead of a module constant. --- pgdevkit/testdb/schema.py | 167 ++++++++++++++++++ .../fixtures/database/app/tables/widget.sql | 4 + .../database/app/tables/widget.test_data.json | 1 + tests/testdb/fixtures/database/schema/app.sql | 1 + tests/testdb/test_schema.py | 60 +++++++ 5 files changed, 233 insertions(+) create mode 100644 pgdevkit/testdb/schema.py create mode 100644 tests/testdb/fixtures/database/app/tables/widget.sql create mode 100644 tests/testdb/fixtures/database/app/tables/widget.test_data.json create mode 100644 tests/testdb/fixtures/database/schema/app.sql create mode 100644 tests/testdb/test_schema.py diff --git a/pgdevkit/testdb/schema.py b/pgdevkit/testdb/schema.py new file mode 100644 index 0000000..1a23046 --- /dev/null +++ b/pgdevkit/testdb/schema.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import json +import logging +import os +import re +from pathlib import Path +from typing import Any, cast + +import psycopg +import sqlglot +import sqlglot.expressions as exp +from psycopg.rows import dict_row +from psycopg.sql import SQL, Identifier, Placeholder + +logger = logging.getLogger(__name__) +logging.getLogger("sqlglot").setLevel(logging.ERROR) + +_TYPE_ORDER = { + "schema": 1, + "types": 2, + "tables": 3, + "scalar_functions": 4, + "functions": 5, + "views": 6, + "table_functions": 7, + "procedures": 8, + "permissions": 100, + "indexes": 101, +} + + +def _get_type_order(path: Path) -> int: + filename = re.sub(r"^\d+(\.\d+)?", "", path.name).removeprefix("_").removesuffix(".sql") + if filename in _TYPE_ORDER: + return _TYPE_ORDER[filename] + if path.parent.name in _TYPE_ORDER: + return _TYPE_ORDER[path.parent.name] + raise ValueError(f"Unknown SQL type for {path.name} in {path.parent.name}") + + +def _get_sql_deps(sql: str) -> tuple[set[str], set[str]]: + exprs = sqlglot.parse(sql, dialect="postgres") + deps: set[str] = set() + declares: set[str] = set() + for e in exprs: + if e is None: + continue + for t in e.find_all(exp.Create): + if t.args.get("this") is not None and t.args.get("db") is not None: + declares.add(str(t)) + for t in e.find_all(exp.Table): + if t.args.get("this") is not None and t.args.get("db") is not None: + deps.add(str(t)) + return declares, deps + + +def _iter_sql_files(database_dir: Path): + """Yield (Path, sql_content) pairs in dependency-safe execution order.""" + files: list[Path] = [] + for root, _, dbfiles in os.walk(database_dir): + if "_migration_scripts" in root or "migrations" in root: + continue + for file in dbfiles: + if file in ("all.sql", "100_permissions.sql"): + continue + if file.endswith(".sql") and ".prod" not in file: + files.append(Path(root) / file) + + delivered_tables: set[str] = set() + delayed: list[tuple[str | None, Path, str]] = [] + all_declared: set[str] = set() + + for file in sorted(files, key=lambda p: (_get_type_order(p), p.name)): + content = file.read_text(encoding="utf-8") + declares, deps = _get_sql_deps(content) + if file.parent.parent.name == "tables": + schema = file.parent.name + full_name = f"{schema}.{file.stem}" + declares.add(full_name) + all_declared.update(declares) + if not deps or all(d in delivered_tables for d in deps): + delivered_tables.add(full_name) + yield file, content + else: + delayed.append((full_name, file, content)) + continue + if not deps or all(d in delivered_tables for d in deps): + yield file, content + else: + delayed.append((None, file, content)) + + while delayed: + progressed = False + for i in range(len(delayed) - 1, -1, -1): + tbl_name, file, content = delayed[i] + _, deps = _get_sql_deps(content) + if all(d in delivered_tables or d not in all_declared for d in deps): + if tbl_name: + delivered_tables.add(tbl_name) + yield file, content + delayed.pop(i) + progressed = True + if not progressed: + raise ValueError(f"Circular or missing SQL dependencies: {[f[1] for f in delayed]}") + + +async def _insert_test_data( + json_file: Path, table: str, force_reset: bool, con: psycopg.AsyncConnection +) -> None: + if not json_file.exists(): + return + rows: list[dict[str, Any]] = json.loads(json_file.read_text(encoding="utf-8")) + if not rows: + return + + schema, table_name = table.split(".") + async with con.cursor(row_factory=dict_row) as cur: + if not force_reset: + await cur.execute(SQL("SELECT count(*) AS cnt FROM {t}").format(t=Identifier(schema, table_name))) + row = await cur.fetchone() + if row and row["cnt"] == len(rows): + return + + col_names = list(rows[0].keys()) + for row in rows: + for col in col_names: + if isinstance(row[col], (dict, list)): + row[col] = json.dumps(row[col]) + + await cur.execute(SQL("DELETE FROM {t}").format(t=Identifier(schema, table_name))) + insert_sql = SQL("INSERT INTO {t} ({cols}) VALUES ({vals})").format( + t=Identifier(schema, table_name), + cols=SQL(", ").join(Identifier(c) for c in col_names), + vals=SQL(", ").join(Placeholder(c) for c in col_names), + ) + await cur.executemany(insert_sql, rows) + + +async def apply_schema( + con: psycopg.AsyncConnection, + database_dir: Path, + extensions: tuple[str, ...] = (), + force_reset: bool = False, +) -> None: + """Apply every .sql file under database_dir (in dependency-safe order) + and seed any matching .test_data.json files. Safe to call repeatedly.""" + await con.set_autocommit(True) + for extension in extensions: + await con.execute(SQL("CREATE EXTENSION IF NOT EXISTS {e}").format(e=Identifier(extension))) + + failures: list[tuple[Path, str]] = [] + for file, sql in _iter_sql_files(database_dir): + try: + await con.execute(cast(Any, sql)) + json_file = file.with_suffix(".test_data.json") + if json_file.exists(): + schema_name = file.parent.parent.name + if re.match(r"^\d+_", schema_name): + schema_name = schema_name.split("_", 1)[1] + await _insert_test_data(json_file, f"{schema_name}.{file.stem}", force_reset, con) + except Exception as e: # noqa: BLE001 + logger.warning("Error executing %s (will retry): %s", file, e) + failures.append((file, sql)) + + for file, sql in failures: + await con.execute(cast(Any, sql)) diff --git a/tests/testdb/fixtures/database/app/tables/widget.sql b/tests/testdb/fixtures/database/app/tables/widget.sql new file mode 100644 index 0000000..3c31956 --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS app.widget ( + id serial PRIMARY KEY, + name text NOT NULL +); diff --git a/tests/testdb/fixtures/database/app/tables/widget.test_data.json b/tests/testdb/fixtures/database/app/tables/widget.test_data.json new file mode 100644 index 0000000..dbd8dc2 --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget.test_data.json @@ -0,0 +1 @@ +[{"id": 1, "name": "sprocket"}] diff --git a/tests/testdb/fixtures/database/schema/app.sql b/tests/testdb/fixtures/database/schema/app.sql new file mode 100644 index 0000000..adc7f94 --- /dev/null +++ b/tests/testdb/fixtures/database/schema/app.sql @@ -0,0 +1 @@ +CREATE SCHEMA IF NOT EXISTS app; diff --git a/tests/testdb/test_schema.py b/tests/testdb/test_schema.py new file mode 100644 index 0000000..f857e4f --- /dev/null +++ b/tests/testdb/test_schema.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from pathlib import Path + +import psycopg +import pytest + +from pgdevkit.testdb import constants +from pgdevkit.testdb.container import ensure_container +from pgdevkit.testdb.schema import apply_schema +from tests.testdb.conftest import requires_podman + +FIXTURES = Path(__file__).parent / "fixtures" / "database" +TEST_DB = "pgdevkit_schema_selftest" + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +def _db_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{TEST_DB}" + + +@pytest.fixture +def schema_test_db(): + ensure_container() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + con.execute(f'CREATE DATABASE "{TEST_DB}"') + yield + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + + +@requires_podman +async def test_apply_schema_creates_tables_and_seeds_data(schema_test_db): + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES) + async with con.cursor() as cur: + await cur.execute("SELECT id, name FROM app.widget ORDER BY id") + rows = await cur.fetchall() + assert rows == [(1, "sprocket")] + + +@requires_podman +async def test_apply_schema_is_idempotent(schema_test_db): + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES) + await apply_schema(con, FIXTURES) # must not raise + async with con.cursor() as cur: + await cur.execute("SELECT count(*) FROM app.widget") + (count,) = await cur.fetchone() + assert count == 1 + + +@requires_podman +async def test_apply_schema_on_missing_directory_is_a_noop(schema_test_db): + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES.parent / "does_not_exist") # must not raise From df14f3f545919adda74d6fa26bceedc02a560730 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 18:45:41 +0200 Subject: [PATCH 07/15] Fix table-dependency tracking in schema apply's SQL walker _iter_sql_files checked file.parent.parent.name == "tables" to detect table files, but for the documented database/{schema}/tables/{name}.sql layout the parent is "tables" and the grandparent is the schema name -- backwards. This meant the table-dependency-tracking branch never fired, so delivered_tables/all_declared stayed empty and files were applied in plain filename order instead of FK dependency order. Fixing the parent/grandparent check alone exposed a second bug: sqlglot's exp.Table walk in _get_sql_deps also matches a CREATE TABLE statement's own target, so every table file's dependency set includes itself. Once dependency tracking actually activates, that self-reference can never be satisfied, deadlocking the delayed-resolution loop. Fixed by discarding the file's own qualified name from its computed deps before checking. Also extracted the schema-name leading-digit-prefix stripping (already present, correctly, in the .test_data.json seeding path) into a shared _strip_layer_prefix() helper used by both the table-dependency branch and the seeding path. Added a regression fixture (widget -> widget_part -> widget_part_detail, a 3-table FK chain whose alphabetical filename order matches its real dependency order) and confirmed empirically that this specific fixture raises UndefinedTable pre-fix and applies cleanly post-fix. A simpler 2-table fixture (gadget referencing widget, sorting before it alphabetically) was tried first per a reviewer's suggestion but was found to pass even against the unfixed code, due to a coincidental interaction between the always-empty all_declared set and the delayed-list's reverse-iteration order -- it does not distinguish correct dependency resolution from the bug for any chain length where alphabetical order is simply the reverse of dependency order. --- pgdevkit/testdb/schema.py | 19 +++++++++++---- .../database/app/tables/widget_part.sql | 4 ++++ .../app/tables/widget_part_detail.sql | 4 ++++ tests/testdb/test_schema.py | 23 +++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 tests/testdb/fixtures/database/app/tables/widget_part.sql create mode 100644 tests/testdb/fixtures/database/app/tables/widget_part_detail.sql diff --git a/pgdevkit/testdb/schema.py b/pgdevkit/testdb/schema.py index 1a23046..a6fc571 100644 --- a/pgdevkit/testdb/schema.py +++ b/pgdevkit/testdb/schema.py @@ -39,6 +39,14 @@ def _get_type_order(path: Path) -> int: raise ValueError(f"Unknown SQL type for {path.name} in {path.parent.name}") +def _strip_layer_prefix(schema_name: str) -> str: + """Strip a leading numeric layer prefix (e.g. "1_dim" -> "dim") so it + matches the unprefixed schema name used in SQL identifiers.""" + if re.match(r"^\d+_", schema_name): + return schema_name.split("_", 1)[1] + return schema_name + + def _get_sql_deps(sql: str) -> tuple[set[str], set[str]]: exprs = sqlglot.parse(sql, dialect="postgres") deps: set[str] = set() @@ -74,9 +82,10 @@ def _iter_sql_files(database_dir: Path): for file in sorted(files, key=lambda p: (_get_type_order(p), p.name)): content = file.read_text(encoding="utf-8") declares, deps = _get_sql_deps(content) - if file.parent.parent.name == "tables": - schema = file.parent.name + if file.parent.name == "tables": + schema = _strip_layer_prefix(file.parent.parent.name) full_name = f"{schema}.{file.stem}" + deps.discard(full_name) # the file's own CREATE TABLE target is not a real dependency declares.add(full_name) all_declared.update(declares) if not deps or all(d in delivered_tables for d in deps): @@ -95,6 +104,8 @@ def _iter_sql_files(database_dir: Path): for i in range(len(delayed) - 1, -1, -1): tbl_name, file, content = delayed[i] _, deps = _get_sql_deps(content) + if tbl_name: + deps.discard(tbl_name) if all(d in delivered_tables or d not in all_declared for d in deps): if tbl_name: delivered_tables.add(tbl_name) @@ -155,9 +166,7 @@ async def apply_schema( await con.execute(cast(Any, sql)) json_file = file.with_suffix(".test_data.json") if json_file.exists(): - schema_name = file.parent.parent.name - if re.match(r"^\d+_", schema_name): - schema_name = schema_name.split("_", 1)[1] + schema_name = _strip_layer_prefix(file.parent.parent.name) await _insert_test_data(json_file, f"{schema_name}.{file.stem}", force_reset, con) except Exception as e: # noqa: BLE001 logger.warning("Error executing %s (will retry): %s", file, e) diff --git a/tests/testdb/fixtures/database/app/tables/widget_part.sql b/tests/testdb/fixtures/database/app/tables/widget_part.sql new file mode 100644 index 0000000..c32a306 --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget_part.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS app.widget_part ( + id serial PRIMARY KEY, + widget_id integer NOT NULL REFERENCES app.widget(id) +); diff --git a/tests/testdb/fixtures/database/app/tables/widget_part_detail.sql b/tests/testdb/fixtures/database/app/tables/widget_part_detail.sql new file mode 100644 index 0000000..1f1cf3b --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget_part_detail.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS app.widget_part_detail ( + id serial PRIMARY KEY, + widget_part_id integer NOT NULL REFERENCES app.widget_part(id) +); diff --git a/tests/testdb/test_schema.py b/tests/testdb/test_schema.py index f857e4f..4f00ed0 100644 --- a/tests/testdb/test_schema.py +++ b/tests/testdb/test_schema.py @@ -58,3 +58,26 @@ async def test_apply_schema_is_idempotent(schema_test_db): async def test_apply_schema_on_missing_directory_is_a_noop(schema_test_db): async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: await apply_schema(con, FIXTURES.parent / "does_not_exist") # must not raise + + +@requires_podman +async def test_apply_schema_resolves_multi_level_fk_dependency(schema_test_db): + # widget_part_detail REFERENCES widget_part REFERENCES widget, and their + # filenames already sort alphabetically in that same dependency order. + # This is the layout that exposed a real bug: _iter_sql_files derived the + # schema/table name from the wrong path component (file.parent.parent.name + # instead of file.parent.name for a database/{schema}/tables/{name}.sql + # layout), so table dependency tracking never activated and files were + # applied in filename order, which for this fixture is the exact reverse + # of the required FK order. + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES) # must not raise + async with con.cursor() as cur: + await cur.execute("INSERT INTO app.widget_part (widget_id) VALUES (1) RETURNING id") + (part_id,) = await cur.fetchone() + await cur.execute( + "INSERT INTO app.widget_part_detail (widget_part_id) VALUES (%s) RETURNING id", + (part_id,), + ) + (detail_id,) = await cur.fetchone() + assert detail_id == 1 From 26c053930c0a1ca2a797778f1b6703728c6c8c09 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 18:55:37 +0200 Subject: [PATCH 08/15] Add pgdevkit.testdb top-level API (ensure/reset/clean/run_sql/status) --- pgdevkit/testdb/__init__.py | 3 + pgdevkit/testdb/api.py | 137 ++++++++++++++++++++++++++++++++++++ tests/testdb/conftest.py | 33 +++++++++ tests/testdb/test_api.py | 81 +++++++++++++++++++++ 4 files changed, 254 insertions(+) create mode 100644 pgdevkit/testdb/api.py create mode 100644 tests/testdb/test_api.py diff --git a/pgdevkit/testdb/__init__.py b/pgdevkit/testdb/__init__.py index e69de29..102da2a 100644 --- a/pgdevkit/testdb/__init__.py +++ b/pgdevkit/testdb/__init__.py @@ -0,0 +1,3 @@ +from .api import clean_testdb, dsn_for, ensure_testdb, reset_testdb, run_sql, status + +__all__ = ["clean_testdb", "dsn_for", "ensure_testdb", "reset_testdb", "run_sql", "status"] diff --git a/pgdevkit/testdb/api.py b/pgdevkit/testdb/api.py new file mode 100644 index 0000000..22fd108 --- /dev/null +++ b/pgdevkit/testdb/api.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +import psycopg + +from . import constants, query +from .config import ProjectConfig, load_config +from .container import ensure_container +from .naming import current_branch, slugify, workspace_db_name +from .schema import apply_schema + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres?connect_timeout=10" + + +def _db_dsn(db_name: str) -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{db_name}?connect_timeout=10" + + +def _resolve(project_root: Path | None) -> tuple[ProjectConfig, str]: + config = load_config(project_root) + branch = current_branch(config.root) + db_name = workspace_db_name(config.name, branch) + return config, db_name + + +def _env_for(config: ProjectConfig, db_name: str) -> dict[str, str]: + prefix = config.env_prefix + return { + f"{prefix}POSTGRES_HOST": constants.HOST, + f"{prefix}POSTGRES_PORT": str(constants.PORT), + f"{prefix}POSTGRES_DB": db_name, + f"{prefix}POSTGRES_USER": constants.USER, + f"{prefix}POSTGRES_PASSWORD": constants.PASSWORD, + } + + +async def _ensure_database(db_name: str) -> None: + async with await psycopg.AsyncConnection.connect(_admin_dsn(), autocommit=True) as con: + result = await con.execute("SELECT 1 FROM pg_database WHERE datname = %(db)s", {"db": db_name}) + if await result.fetchone(): + return + try: + await con.execute(f'CREATE DATABASE "{db_name}"') + except psycopg.errors.DuplicateDatabase: + pass + + +async def _drop_database(db_name: str) -> None: + async with await psycopg.AsyncConnection.connect(_admin_dsn(), autocommit=True) as con: + await con.execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = %(db)s", + {"db": db_name}, + ) + await con.execute(f'DROP DATABASE IF EXISTS "{db_name}"') + + +async def _apply(config: ProjectConfig, db_name: str, force_reset: bool) -> None: + async with await psycopg.AsyncConnection.connect(_db_dsn(db_name), autocommit=True) as con: + await apply_schema( + con, + config.root / config.database_dir, + extensions=config.extensions, + force_reset=force_reset, + ) + + +def ensure_testdb(project_root: Path | None = None, force_reset: bool = False) -> dict[str, str]: + """Ensure the shared container is running, this workspace's database + exists, and its schema is applied. Returns the {PREFIX}POSTGRES_* env + vars for this workspace.""" + if not os.environ.get("PGDEVKIT_SKIP_CONTAINER"): + ensure_container() + config, db_name = _resolve(project_root) + + async def _run() -> None: + if force_reset: + await _drop_database(db_name) + await _ensure_database(db_name) + await _apply(config, db_name, force_reset) + + asyncio.run(_run()) + return _env_for(config, db_name) + + +def reset_testdb(project_root: Path | None = None) -> dict[str, str]: + """Drop and recreate only this workspace's database, then reapply + schema and seed data.""" + return ensure_testdb(project_root, force_reset=True) + + +def clean_testdb(project_root: Path | None = None, all: bool = False) -> None: + """Drop this workspace's database. With all=True, drop every database + belonging to this project (matched by its name-slug prefix), across + every worktree/branch.""" + config, db_name = _resolve(project_root) + + async def _run() -> None: + if not all: + await _drop_database(db_name) + return + prefix = f"{slugify(config.name)}_" + async with await psycopg.AsyncConnection.connect(_admin_dsn(), autocommit=True) as con: + result = await con.execute( + "SELECT datname FROM pg_database WHERE datname LIKE %(pattern)s", + {"pattern": f"{prefix}%"}, + ) + names = [row[0] for row in await result.fetchall()] + for name in names: + await _drop_database(name) + + asyncio.run(_run()) + + +def status(project_root: Path | None = None) -> dict[str, str]: + config, db_name = _resolve(project_root) + return { + "container": constants.CONTAINER_NAME, + "host": constants.HOST, + "port": str(constants.PORT), + "database": db_name, + "dsn": _db_dsn(db_name), + } + + +def run_sql(sql: str, project_root: Path | None = None) -> list[dict] | None: + _, db_name = _resolve(project_root) + return asyncio.run(query.execute(_db_dsn(db_name), sql)) + + +def dsn_for(project_root: Path | None = None) -> str: + _, db_name = _resolve(project_root) + return _db_dsn(db_name) diff --git a/tests/testdb/conftest.py b/tests/testdb/conftest.py index 23f9a24..e7f118a 100644 --- a/tests/testdb/conftest.py +++ b/tests/testdb/conftest.py @@ -1,9 +1,42 @@ from __future__ import annotations import shutil +import subprocess +from pathlib import Path +from typing import Callable import pytest requires_podman = pytest.mark.skipif( shutil.which("podman") is None, reason="podman is not installed" ) + +FIXTURES = Path(__file__).parent / "fixtures" / "database" + + +def _make_project(base: Path, name: str, branch: str) -> Path: + project = base / f"{name}-{branch}" + project.mkdir() + (project / "database").symlink_to(FIXTURES) + (project / "pyproject.toml").write_text( + f'[tool.pgdevkit]\nname = "{name}"\n', encoding="utf-8" + ) + for cmd in ( + ["git", "init", "-q"], + ["git", "config", "user.email", "test@example.com"], + ["git", "config", "user.name", "test"], + ["git", "checkout", "-q", "-b", branch], + ): + subprocess.run(cmd, cwd=project, check=True) + (project / ".gitkeep").write_text("", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=project, check=True) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=project, check=True) + return project + + +@pytest.fixture +def project_factory(tmp_path: Path) -> Callable[[str, str], Path]: + def _factory(name: str, branch: str) -> Path: + return _make_project(tmp_path, name, branch) + + return _factory diff --git a/tests/testdb/test_api.py b/tests/testdb/test_api.py new file mode 100644 index 0000000..b2f472c --- /dev/null +++ b/tests/testdb/test_api.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +import psycopg +import pytest + +from pgdevkit.testdb import constants +from pgdevkit.testdb.api import clean_testdb, ensure_testdb, reset_testdb, status +from tests.testdb.conftest import requires_podman + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +@requires_podman +def test_ensure_testdb_isolates_by_branch(project_factory: Callable[[str, str], Path]): + project_a = project_factory("apitest", "main") + project_b = project_factory("apitest", "feature") + try: + env_a = ensure_testdb(project_a) + env_b = ensure_testdb(project_b) + + assert env_a["APITEST_POSTGRES_DB"] != env_b["APITEST_POSTGRES_DB"] + assert env_a["APITEST_POSTGRES_DB"] == status(project_a)["database"] + finally: + clean_testdb(project_a) + clean_testdb(project_b) + + +@requires_podman +def test_clean_all_removes_every_branch_database(project_factory: Callable[[str, str], Path]): + project_a = project_factory("apitest2", "main") + project_b = project_factory("apitest2", "feature") + ensure_testdb(project_a) + ensure_testdb(project_b) + + clean_testdb(project_a, all=True) + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute("SELECT count(*) FROM pg_database WHERE datname LIKE 'apitest2_%'") + (count,) = cur.fetchone() + assert count == 0 + + +@requires_podman +def test_reset_testdb_only_touches_own_database(project_factory: Callable[[str, str], Path]): + project_a = project_factory("apitest3", "main") + project_b = project_factory("apitest3", "other") + try: + ensure_testdb(project_a) + ensure_testdb(project_b) + + reset_testdb(project_a) # must not raise or affect project_b + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_database WHERE datname = %s", + (status(project_b)["database"],), + ) + (count,) = cur.fetchone() + assert count == 1 + finally: + clean_testdb(project_a) + clean_testdb(project_b) + + +@requires_podman +def test_dsn_for_matches_status(project_factory: Callable[[str, str], Path]): + from pgdevkit.testdb.api import dsn_for + + project = project_factory("apitest4", "main") + try: + ensure_testdb(project) + assert dsn_for(project) == status(project)["dsn"] + finally: + clean_testdb(project) From f96bd4fb25c813ce5c660d95a4458ed5d388c5c1 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 19:15:52 +0200 Subject: [PATCH 09/15] Fix clean_testdb(all=True) LIKE-wildcard cross-project match The all-databases prefix pattern used the project's slug directly in a SQL LIKE pattern without escaping "_", which LIKE treats as a single-character wildcard. This let clean_testdb(all=True) for a project like "app" also match and drop a differently-named project's databases (e.g. "apps"), violating the "never touch another project's databases" guarantee. Escape LIKE metacharacters in the prefix and add a regression test with slug-colliding project names. --- pgdevkit/testdb/api.py | 5 +++-- tests/testdb/test_api.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/pgdevkit/testdb/api.py b/pgdevkit/testdb/api.py index 22fd108..233ccb4 100644 --- a/pgdevkit/testdb/api.py +++ b/pgdevkit/testdb/api.py @@ -104,10 +104,11 @@ async def _run() -> None: await _drop_database(db_name) return prefix = f"{slugify(config.name)}_" + escaped_prefix = prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") async with await psycopg.AsyncConnection.connect(_admin_dsn(), autocommit=True) as con: result = await con.execute( - "SELECT datname FROM pg_database WHERE datname LIKE %(pattern)s", - {"pattern": f"{prefix}%"}, + "SELECT datname FROM pg_database WHERE datname LIKE %(pattern)s ESCAPE '\\'", + {"pattern": f"{escaped_prefix}%"}, ) names = [row[0] for row in await result.fetchall()] for name in names: diff --git a/tests/testdb/test_api.py b/tests/testdb/test_api.py index b2f472c..c06dd87 100644 --- a/tests/testdb/test_api.py +++ b/tests/testdb/test_api.py @@ -46,6 +46,36 @@ def test_clean_all_removes_every_branch_database(project_factory: Callable[[str, assert count == 0 +@requires_podman +def test_clean_all_does_not_match_prefix_colliding_project_name( + project_factory: Callable[[str, str], Path], +): + # "apitestx" and "apitestxs" collide under an unescaped LIKE pattern: the + # pattern "apitestx_%" (built from the "apitestx_" prefix) would also + # match "apitestxs_main" because "_" is a single-character SQL wildcard + # that consumes the "s". clean_testdb(..., all=True) for "apitestx" must + # never touch "apitestxs"'s database. + project_short = project_factory("apitestx", "main") + project_long = project_factory("apitestxs", "main") + try: + ensure_testdb(project_short) + ensure_testdb(project_long) + + clean_testdb(project_short, all=True) + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_database WHERE datname = %s", + (status(project_long)["database"],), + ) + (count,) = cur.fetchone() + assert count == 1 + finally: + clean_testdb(project_short) + clean_testdb(project_long) + + @requires_podman def test_reset_testdb_only_touches_own_database(project_factory: Callable[[str, str], Path]): project_a = project_factory("apitest3", "main") From 32470023357ca920862ac3c943be3dcbe06495d7 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 19:32:19 +0200 Subject: [PATCH 10/15] Add pgdb testdb CLI (up/reset/run-sql/status/shell/clean) --- pgdevkit/cli.py | 73 +++++++++++++++++++++++++++++++++++++++- tests/testdb/test_cli.py | 63 ++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 tests/testdb/test_cli.py diff --git a/pgdevkit/cli.py b/pgdevkit/cli.py index 00ec2be..e6d1390 100644 --- a/pgdevkit/cli.py +++ b/pgdevkit/cli.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from pathlib import Path import typer @@ -7,6 +8,7 @@ from rich.table import Table from rich import box +from . import testdb from .connection import build_conninfo from .diff import DiffKind, compute_diff from .introspect import introspect_db @@ -16,10 +18,12 @@ console = Console() err_console = Console(stderr=True) +testdb_app = typer.Typer(name="testdb", help="Manage the shared local Postgres test container") +app.add_typer(testdb_app, name="testdb") + @app.command() def compare( - url: str = typer.Option(..., "--url", help="PostgreSQL DSN (postgresql://user:pass@host:port/db)"), entra_user: str | None = typer.Option(None, "--entra-user", help="Azure Entra user (triggers token auth)"), report_extra_db: bool = typer.Option(False, "--report-extra-db", help="Report objects in DB but not in scripts"), @@ -61,3 +65,70 @@ def compare( console.print(table) console.print(f"\n[bold red]{len(diffs)} difference(s) found.[/bold red]") raise typer.Exit(1) + + +@testdb_app.command("up") +def testdb_up() -> None: + """Ensure the container is running, the workspace DB exists, and schema is applied.""" + testdb.ensure_testdb() + info = testdb.status() + console.print(f"[green]Test DB ready:[/green] {info['database']} ({info['dsn']})") + + +@testdb_app.command("reset") +def testdb_reset() -> None: + """Drop and recreate only this workspace's database, then reapply schema + seed data.""" + testdb.reset_testdb() + info = testdb.status() + console.print(f"[green]Test DB reset:[/green] {info['database']}") + + +@testdb_app.command("run-sql") +def testdb_run_sql( + file: Path | None = typer.Argument(None, help="Path to a .sql file"), + sql: str | None = typer.Option(None, "--sql", help="Inline SQL string"), + results: bool = typer.Option(False, "--results", help="Print query results as a table"), +) -> None: + """Run SQL against this workspace's database.""" + if (file is None) == (sql is None): + err_console.print("[red]Error:[/red] pass exactly one of FILE or --sql") + raise typer.Exit(2) + statement = file.read_text(encoding="utf-8") if file else sql + assert statement is not None + rows = testdb.run_sql(statement) + + if rows is None: + console.print("OK") + return + if not results: + console.print(f"OK — {len(rows)} row(s)") + return + table = Table(box=box.SIMPLE, show_header=True, header_style="bold") + for col in rows[0]: + table.add_column(col) + for row in rows: + table.add_row(*(str(v) for v in row.values())) + console.print(table) + console.print(f"({len(rows)} row(s))") + + +@testdb_app.command("status") +def testdb_status() -> None: + """Show container state, this workspace's database name, and DSN.""" + for key, value in testdb.status().items(): + console.print(f"{key}: {value}") + + +@testdb_app.command("shell") +def testdb_shell() -> None: + """Drop into psql against this workspace's database.""" + os.execvp("psql", ["psql", testdb.dsn_for()]) + + +@testdb_app.command("clean") +def testdb_clean( + all: bool = typer.Option(False, "--all", help="Drop every database belonging to this project"), +) -> None: + """Drop this workspace's database (or every database of this project with --all).""" + testdb.clean_testdb(all=all) + console.print("[green]Cleaned.[/green]") diff --git a/tests/testdb/test_cli.py b/tests/testdb/test_cli.py new file mode 100644 index 0000000..33215cb --- /dev/null +++ b/tests/testdb/test_cli.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from typer.testing import CliRunner + +from pgdevkit.cli import app +from pgdevkit.testdb import clean_testdb +from tests.testdb.conftest import requires_podman + +runner = CliRunner() + + +@requires_podman +def test_testdb_up_and_status(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest", "main") + monkeypatch.chdir(project) + try: + result = runner.invoke(app, ["testdb", "up"]) + assert result.exit_code == 0, result.output + + result = runner.invoke(app, ["testdb", "status"]) + assert result.exit_code == 0, result.output + assert "database:" in result.output + finally: + clean_testdb(project) + + +@requires_podman +def test_testdb_run_sql_inline_with_results(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest2", "main") + monkeypatch.chdir(project) + try: + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke( + app, ["testdb", "run-sql", "--sql", "SELECT id, name FROM app.widget", "--results"] + ) + assert result.exit_code == 0, result.output + assert "sprocket" in result.output + finally: + clean_testdb(project) + + +@requires_podman +def test_testdb_reset(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest3", "main") + monkeypatch.chdir(project) + try: + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke(app, ["testdb", "reset"]) + assert result.exit_code == 0, result.output + finally: + clean_testdb(project) + + +@requires_podman +def test_testdb_clean(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest4", "main") + monkeypatch.chdir(project) + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke(app, ["testdb", "clean"]) + assert result.exit_code == 0, result.output From 490662335cea51a4109793ed253ddd7db44a3862 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 19:43:17 +0200 Subject: [PATCH 11/15] Fix IndexError in testdb run-sql --results on zero-row SELECT --- pgdevkit/cli.py | 3 +++ tests/testdb/test_cli.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/pgdevkit/cli.py b/pgdevkit/cli.py index e6d1390..2879978 100644 --- a/pgdevkit/cli.py +++ b/pgdevkit/cli.py @@ -103,6 +103,9 @@ def testdb_run_sql( if not results: console.print(f"OK — {len(rows)} row(s)") return + if not rows: + console.print("(0 rows)") + return table = Table(box=box.SIMPLE, show_header=True, header_style="bold") for col in rows[0]: table.add_column(col) diff --git a/tests/testdb/test_cli.py b/tests/testdb/test_cli.py index 33215cb..67bd12b 100644 --- a/tests/testdb/test_cli.py +++ b/tests/testdb/test_cli.py @@ -42,6 +42,21 @@ def test_testdb_run_sql_inline_with_results(project_factory: Callable[[str, str] clean_testdb(project) +@requires_podman +def test_testdb_run_sql_with_results_and_zero_rows(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest5", "main") + monkeypatch.chdir(project) + try: + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke( + app, ["testdb", "run-sql", "--sql", "SELECT id, name FROM app.widget WHERE false", "--results"] + ) + assert result.exit_code == 0, result.output + assert "0 row" in result.output + finally: + clean_testdb(project) + + @requires_podman def test_testdb_reset(project_factory: Callable[[str, str], Path], monkeypatch): project = project_factory("clitest3", "main") From c0e5e34cf2c5520f8c49dbd6256037cc14434473 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 20:14:59 +0200 Subject: [PATCH 12/15] Dogfood pgdevkit.testdb in pgdevkit's own test suite Replace tests/conftest.py's ad hoc docker-package container/DB bootstrap with a call to pgdevkit.testdb.ensure_testdb(), so pgdevkit's own tests use the same shared Podman-backed container as consumers of the testdb CLI. Removes the now-unused docker dependency and static PGDB_TEST_POSTGRES_* pytest_env entries. Pin [tool.pgdevkit] name = "pgdevkit" explicitly so the env_prefix is deterministic regardless of the checkout directory name (worktrees are named differently from the main clone). Documents pgdb testdb usage in the README. --- README.md | 42 +++++++++++++++++++++++++++ pyproject.toml | 11 ++------ tests/conftest.py | 72 ++++------------------------------------------- uv.lock | 29 ------------------- 4 files changed, 51 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index e69de29..8aaf6e2 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,42 @@ +# pgdevkit + +A helper for developing with Postgres. + +## `pgdb compare` + +Compare a directory of SQL scripts (see the `database-in-source` layout +convention) against a live database and report differences: + +```bash +pgdb compare --url postgresql://user:pass@host:port/db path/to/database/ +``` + +## `pgdb testdb` + +Manages a single shared, Podman-backed Postgres container for local tests +across all your projects — no more one-container-per-project-per-worktree. +Isolation between projects and worktrees is per-database, inside one +container. + +Add to `pyproject.toml`: + +```toml +[tool.pgdevkit] +name = "myproject" # optional; defaults to the repo directory name +database_dir = "database" # optional; defaults to "database" +``` + +Add to `conftest.py`: + +```python +import os +import pytest +from pgdevkit.testdb import ensure_testdb + +@pytest.fixture(scope="session", autouse=True) +def ensure_test_postgres(): + for k, v in ensure_testdb().items(): + os.environ[k] = v +``` + +CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`. diff --git a/pyproject.toml b/pyproject.toml index 4a1ec68..dd5ad2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,9 +26,11 @@ cli = [ [project.scripts] pgdb = "pgdevkit.cli:app" +[tool.pgdevkit] +name = "pgdevkit" + [dependency-groups] test = [ - "docker>=7.1.0", "psycopg[binary]>=3.2.0", "pytest>=9.1.0", "pytest-asyncio>=0.24.0", @@ -38,10 +40,3 @@ test = [ [tool.pytest.ini_options] pythonpath = ["."] asyncio_mode = "auto" - -[tool.pytest_env] -PGDB_TEST_POSTGRES_HOST = "localhost" -PGDB_TEST_POSTGRES_PORT = "54326" -PGDB_TEST_POSTGRES_DB = "pgdb_test" -PGDB_TEST_POSTGRES_USER = "postgres" -PGDB_TEST_POSTGRES_PASSWORD = "testpwd" diff --git a/tests/conftest.py b/tests/conftest.py index 00a8fcf..7bf0783 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,80 +1,20 @@ from __future__ import annotations -import os -import time -from pathlib import Path from typing import Any import psycopg import pytest -TEST_PORT = int(os.environ.get("PGDB_TEST_POSTGRES_PORT", "54326")) -TEST_DB = os.environ.get("PGDB_TEST_POSTGRES_DB", "pgdb_test") -TEST_USER = os.environ.get("PGDB_TEST_POSTGRES_USER", "postgres") -TEST_PASSWORD = os.environ.get("PGDB_TEST_POSTGRES_PASSWORD", "testpwd") -TEST_HOST = os.environ.get("PGDB_TEST_POSTGRES_HOST", "localhost") -CONTAINER_NAME = "pgdb_postgres4test" - -DSN = f"postgresql://{TEST_USER}:{TEST_PASSWORD}@{TEST_HOST}:{TEST_PORT}/{TEST_DB}" - - -def _start_docker() -> None: - import docker # type: ignore - import docker.errors # type: ignore - - client = docker.from_env() - try: - existing = client.containers.get(CONTAINER_NAME) - if existing.status != "running": - existing.start() - return - except docker.errors.NotFound: - pass - - client.containers.run( - "postgres:17", - name=CONTAINER_NAME, - detach=True, - ports={"5432/tcp": TEST_PORT}, - environment={ - "POSTGRES_PASSWORD": TEST_PASSWORD, - "POSTGRES_DB": TEST_DB, - "POSTGRES_USER": TEST_USER, - }, - ) - - -def _wait_for_postgres(timeout: int = 30) -> None: - admin_dsn = f"postgresql://{TEST_USER}:{TEST_PASSWORD}@{TEST_HOST}:{TEST_PORT}/postgres" - for _ in range(timeout): - try: - with psycopg.connect(admin_dsn, connect_timeout=2): - return - except Exception: - time.sleep(1) - raise RuntimeError("Postgres did not become ready in time") - - -def _ensure_test_db() -> None: - from psycopg import sql - - admin_dsn = f"postgresql://{TEST_USER}:{TEST_PASSWORD}@{TEST_HOST}:{TEST_PORT}/postgres" - with psycopg.connect(admin_dsn, autocommit=True) as conn: - exists = conn.execute( - sql.SQL("SELECT 1 FROM pg_database WHERE datname = %s"), (TEST_DB,) - ).fetchone() - if not exists: - conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(TEST_DB))) +from pgdevkit.testdb import ensure_testdb @pytest.fixture(scope="session") def postgres_dsn() -> str: - if os.environ.get("SKIP_START_POSTGRES") == "1": - return DSN - _start_docker() - _wait_for_postgres() - _ensure_test_db() - return DSN + env = ensure_testdb() + return ( + f"postgresql://{env['PGDEVKIT_POSTGRES_USER']}:{env['PGDEVKIT_POSTGRES_PASSWORD']}" + f"@{env['PGDEVKIT_POSTGRES_HOST']}:{env['PGDEVKIT_POSTGRES_PORT']}/{env['PGDEVKIT_POSTGRES_DB']}" + ) @pytest.fixture diff --git a/uv.lock b/uv.lock index 52b2739..be89045 100644 --- a/uv.lock +++ b/uv.lock @@ -182,20 +182,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, ] -[[package]] -name = "docker" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "requests" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, -] - [[package]] name = "idna" version = "3.18" @@ -290,7 +276,6 @@ cli = [ [package.dev-dependencies] test = [ - { name = "docker" }, { name = "psycopg", extra = ["binary"] }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -309,7 +294,6 @@ provides-extras = ["azure", "cli"] [package.metadata.requires-dev] test = [ - { name = "docker", specifier = ">=7.1.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, { name = "pytest", specifier = ">=9.1.0" }, { name = "pytest-asyncio", specifier = ">=0.24.0" }, @@ -442,19 +426,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] -[[package]] -name = "pywin32" -version = "312" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, - { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, - { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, - { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, - { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, - { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, -] - [[package]] name = "requests" version = "2.34.2" From dbb8d506c4bcf679f41cdd2f9333c86bd3cd18a1 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 20:23:35 +0200 Subject: [PATCH 13/15] Gitignore local migration checklist for downstream projects --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e1c7f4c..ecfbe1c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ wheels/ # Local planning docs (not shipped with the package) docs/superpowers/ + +# Local migration checklist (not shipped with the package) +/MIGRATION_NOTES.md From 16c38246335e895cc55d3c152ad777d38a76551c Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Wed, 15 Jul 2026 21:17:50 +0200 Subject: [PATCH 14/15] Fix sqlglot 30.11.0 name-extraction and definition-comparison bugs CREATE SCHEMA and CREATE FUNCTION parse to AST shapes under the pinned sqlglot==30.11.0 that the existing name-extraction in parser.py doesn't handle (Table.name resolves empty when the real name is in .db; UserDefinedFunction.name doesn't unwrap a nested Table), causing spurious "" schema and "public." function entries. Fix both by falling back to the db-arg/nested-Table extraction pattern already used elsewhere in the file. Also fix two more latent bugs of the same kind surfaced by the same test run: _parse_table_constraint referenced exp.UniqueKey/exp.Unique, which don't exist in this sqlglot version (real class is UniqueColumnConstraint), so any table with a FOREIGN KEY or UNIQUE constraint raised and was silently dropped entirely by the per-file exception handler; and _norm_sql didn't strip the trailing ";" that Postgres's pg_get_viewdef()/pg_get_indexdef() always add but sqlglot's .sql() rendering never does, so every structurally-identical view compared as a mismatch. Finally, simplify the active_users view fixture so it no longer selects users.email or filters on users.status -- Postgres refuses ALTER COLUMN TYPE on any column referenced by a view, which blocked two test scenarios. tests/test_compare.py goes from 5/9 failing to 9/9 passing; full suite (39 tests) passes with no regressions. --- pgdevkit/diff.py | 5 ++++- pgdevkit/parser.py | 35 ++++++++++++++++++++++++++--------- tests/fixtures/04_views.sql | 2 +- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/pgdevkit/diff.py b/pgdevkit/diff.py index 9841fb9..390e753 100644 --- a/pgdevkit/diff.py +++ b/pgdevkit/diff.py @@ -204,7 +204,10 @@ def _unwrap_paren(node): def _norm_sql(s: str) -> str: - return " ".join(s.lower().split()) + # Postgres's pg_get_viewdef()/pg_get_indexdef() always append a trailing + # ";", while sqlglot's Expression.sql() rendering never does; strip it so + # semantically-identical definitions compare equal. + return " ".join(s.lower().split()).rstrip(";") def _norm_body(s: str) -> str: diff --git a/pgdevkit/parser.py b/pgdevkit/parser.py index 8142fdb..39b61f7 100644 --- a/pgdevkit/parser.py +++ b/pgdevkit/parser.py @@ -170,7 +170,7 @@ def _parse_table_constraint(item: exp.Expression) -> ConstraintDef | None: if isinstance(inner, exp.PrimaryKey): kind = "PRIMARY KEY" - elif isinstance(inner, (exp.UniqueKey, exp.Unique)): + elif isinstance(inner, exp.UniqueColumnConstraint): kind = "UNIQUE" elif isinstance(inner, exp.ForeignKey): kind = "FOREIGN KEY" @@ -197,15 +197,25 @@ def _handle_view(expr: exp.Create, db_schema: DatabaseSchema) -> None: def _handle_function(expr: exp.Create, raw: str, db_schema: DatabaseSchema, kind: str) -> None: # Get name/schema from sqlglot func_node = expr.this - if hasattr(func_node, "name"): - fname = func_node.name + fname = func_node.name if hasattr(func_node, "name") else "" + if fname: db_node = func_node.args.get("db") if hasattr(func_node, "args") else None fschema = db_node.name if db_node else "public" else: - result = _resolve_name(expr) - if not result: - return - fschema, fname = result + # sqlglot (30.11.0) parses "CREATE FUNCTION myapp.greet(...)" as a + # UserDefinedFunction wrapping a Table (this=Identifier(greet), + # db=Identifier(myapp)). UserDefinedFunction.name doesn't unwrap + # that nested Table, so pull the name/schema from it directly. + inner = func_node.this if hasattr(func_node, "this") else None + if isinstance(inner, exp.Table) and inner.name: + fname = inner.name + db_node = inner.args.get("db") + fschema = db_node.name if db_node else "public" + else: + result = _resolve_name(expr) + if not result: + return + fschema, fname = result # Extract args, return type, language, body with regex on raw SQL args, return_type, language, body = _parse_function_details(raw) @@ -275,8 +285,15 @@ def _handle_type(expr: exp.Create, db_schema: DatabaseSchema) -> None: def _handle_schema_create(expr: exp.Create, db_schema: DatabaseSchema) -> None: this = expr.this - if hasattr(this, "name"): - db_schema.schemas.add(this.name) + name = this.name if hasattr(this, "name") else "" + if not name: + # sqlglot (30.11.0) parses "CREATE SCHEMA myapp" as a Table node + # whose `db` arg holds the schema name and `.name` (the table + # identifier) is empty. + db_node = this.args.get("db") if hasattr(this, "args") else None + name = db_node.name if db_node else "" + if name: + db_schema.schemas.add(name) def _handle_index(expr: exp.Create, db_schema: DatabaseSchema) -> None: diff --git a/tests/fixtures/04_views.sql b/tests/fixtures/04_views.sql index 9092529..3b753ea 100644 --- a/tests/fixtures/04_views.sql +++ b/tests/fixtures/04_views.sql @@ -1,2 +1,2 @@ CREATE OR REPLACE VIEW myapp.active_users AS -SELECT id, email FROM myapp.users WHERE status = 'active'; +SELECT id FROM myapp.users; From 666148aa4b6203b4efaf0ed12f1062703c89e433 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Thu, 16 Jul 2026 07:23:43 +0200 Subject: [PATCH 15/15] Add ty as dev dependency, fix real type errors it caught MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ty check pgdevkit now passes: - config.py: drop the dead tomli fallback (requires-python >= 3.14 guarantees tomllib, already flagged as dead code in review) - api.py: build CREATE/DROP DATABASE via psycopg.sql.SQL/Identifier instead of an f-string, matching psycopg's typed-query API - query.py: cast the caller-provided SQL text to LiteralString with an explanatory comment — this function's whole purpose is running arbitrary SQL, so it can never itself be a literal Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_015bpLodYTShC96vqY6gqsQy --- pgdevkit/testdb/api.py | 5 +++-- pgdevkit/testdb/config.py | 6 +----- pgdevkit/testdb/query.py | 7 ++++++- pyproject.toml | 3 +++ uv.lock | 29 +++++++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/pgdevkit/testdb/api.py b/pgdevkit/testdb/api.py index 233ccb4..b1cbbbb 100644 --- a/pgdevkit/testdb/api.py +++ b/pgdevkit/testdb/api.py @@ -5,6 +5,7 @@ from pathlib import Path import psycopg +from psycopg.sql import SQL, Identifier from . import constants, query from .config import ProjectConfig, load_config @@ -45,7 +46,7 @@ async def _ensure_database(db_name: str) -> None: if await result.fetchone(): return try: - await con.execute(f'CREATE DATABASE "{db_name}"') + await con.execute(SQL("CREATE DATABASE {}").format(Identifier(db_name))) except psycopg.errors.DuplicateDatabase: pass @@ -56,7 +57,7 @@ async def _drop_database(db_name: str) -> None: "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = %(db)s", {"db": db_name}, ) - await con.execute(f'DROP DATABASE IF EXISTS "{db_name}"') + await con.execute(SQL("DROP DATABASE IF EXISTS {}").format(Identifier(db_name))) async def _apply(config: ProjectConfig, db_name: str, force_reset: bool) -> None: diff --git a/pgdevkit/testdb/config.py b/pgdevkit/testdb/config.py index 814e956..79ace7c 100644 --- a/pgdevkit/testdb/config.py +++ b/pgdevkit/testdb/config.py @@ -1,13 +1,9 @@ from __future__ import annotations +import tomllib from dataclasses import dataclass, field from pathlib import Path -try: - import tomllib -except ImportError: # pragma: no cover - Python < 3.11 - import tomli as tomllib # type: ignore[no-redef] - @dataclass(frozen=True) class ProjectConfig: diff --git a/pgdevkit/testdb/query.py b/pgdevkit/testdb/query.py index 163fdc2..bf82f3d 100644 --- a/pgdevkit/testdb/query.py +++ b/pgdevkit/testdb/query.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import LiteralString, cast + import psycopg from psycopg.rows import dict_row @@ -12,6 +14,9 @@ async def execute(dsn: str, sql: str) -> list[dict] | None: async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as con: for stmt in statements: async with con.cursor(row_factory=dict_row) as cur: - await cur.execute(stmt) + # stmt is arbitrary, caller-provided SQL text (a .sql file or + # --sql argument) — not a compile-time literal, but this + # function's entire purpose is to run it as-is. + await cur.execute(cast(LiteralString, stmt)) last_rows = await cur.fetchall() if cur.description else None return last_rows diff --git a/pyproject.toml b/pyproject.toml index dd5ad2e..abf8600 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ pgdb = "pgdevkit.cli:app" name = "pgdevkit" [dependency-groups] +dev = [ + "ty>=0.0.59", +] test = [ "psycopg[binary]>=3.2.0", "pytest>=9.1.0", diff --git a/uv.lock b/uv.lock index be89045..7121098 100644 --- a/uv.lock +++ b/uv.lock @@ -275,6 +275,9 @@ cli = [ ] [package.dev-dependencies] +dev = [ + { name = "ty" }, +] test = [ { name = "psycopg", extra = ["binary"] }, { name = "pytest" }, @@ -293,6 +296,7 @@ requires-dist = [ provides-extras = ["azure", "cli"] [package.metadata.requires-dev] +dev = [{ name = "ty", specifier = ">=0.0.59" }] test = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, { name = "pytest", specifier = ">=9.1.0" }, @@ -492,6 +496,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/e5/d52b6d4e8e06c1a9771eff5886da6a4381aa624d94b5a8cc846f22a2b270/sqlglotc-30.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:8c6ed84d557772880a082ef29d18b0c2cfee2d177e6528b30df2b7eaf6adb585", size = 10984419, upload-time = "2026-06-11T17:10:39.797Z" }, ] +[[package]] +name = "ty" +version = "0.0.59" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/b0/84ae7b3bf6e3e9f57eb9635eeff5a80b36e57aa089f40be0fb5c384fa176/ty-0.0.59.tar.gz", hash = "sha256:53e53ffeed78ad59cd237fa8ea1316d2b94e13efdea9a945698acab549e005aa", size = 6145435, upload-time = "2026-07-12T20:22:02.781Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/e8/650b42fbef4d48e6ca682b0b6e9b68fa8fcf55cbb0a6892ab89990018b6f/ty-0.0.59-py3-none-linux_armv6l.whl", hash = "sha256:f8fb08a767ef8f11ea3c537b9d77860726cc2bc39e6f77ad13c02d5b289f20a7", size = 11700328, upload-time = "2026-07-12T20:21:26.046Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/0ca3a89d5f59ae5f308e5e83428cac5f9143200767743e052fba90b4b81e/ty-0.0.59-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c7f4d5630836c8a0ba13dd4ac7bdae080a7d6ebe965b817ff642dc961bcf2a53", size = 11494310, upload-time = "2026-07-12T20:21:28.491Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f8/5076de6001cefbccd8e6dc8472262697e43308ff66b0e87c72abba136357/ty-0.0.59-py3-none-macosx_11_0_arm64.whl", hash = "sha256:872f6fb02c6db5553c4d5fb283b3d50f0985fb9a29a910e4fda4793a775c1926", size = 11026797, upload-time = "2026-07-12T20:21:30.879Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0f/fca28481b6a138e2b798ad9fdc98a095475f9104948ba242fce4b477782b/ty-0.0.59-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2af8eefbfe806337770eec12c0c819c5f1b8f5b85f8369cb1cc9fa25234a2208", size = 11475304, upload-time = "2026-07-12T20:21:33.041Z" }, + { url = "https://files.pythonhosted.org/packages/08/4b/1fed8b81b389ef4bbc0400f19e05fc16496b162577779dc0e5fc65ac216c/ty-0.0.59-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0acf8b76a1c9a7ddef460b42475f6c76193164426ab080783af1c3175b4b999b", size = 11533131, upload-time = "2026-07-12T20:21:35.189Z" }, + { url = "https://files.pythonhosted.org/packages/5f/fc/04eec35e05a10e0fea1c6503a290ccc3935efda9c845aff64e83282c1af7/ty-0.0.59-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:043c2e00eb1d7475f928af7dedd71f69b64e69bfca55e36f4c968479e1373fc4", size = 12205932, upload-time = "2026-07-12T20:21:37.324Z" }, + { url = "https://files.pythonhosted.org/packages/a9/dd/a61de859659fa11b55917ad38340a8f2c61f5ae17d1874929f29084c6990/ty-0.0.59-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0d688d857441df57f48fca66c029d85cf737c510e7be1d01144cdad1e58d968", size = 12758406, upload-time = "2026-07-12T20:21:39.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/e8/fa66f05997eab8ca75fc4f17320140e25467849e0cc75597f898cc22099c/ty-0.0.59-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a96c9f88394a3b42c737e2125b2330543f0d90a43b49761f377d96f8c3ee0d62", size = 12288176, upload-time = "2026-07-12T20:21:41.784Z" }, + { url = "https://files.pythonhosted.org/packages/15/68/0fca59963bd5123f42d5f7da50667e7a52e8e9615e3a16d8c2c0d3b2d143/ty-0.0.59-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08dbcb268edcafcb152e59475b5b495ce28d0b340a395c09943557678f4d5a6", size = 12028471, upload-time = "2026-07-12T20:21:43.82Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5b/cd7dabbbab392578f11179919da5c25d8c3322e5388a688f539ea0539603/ty-0.0.59-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8812764b9a40fdc98df1272826e73a298ef56b06681135e643bcf90aad1896f7", size = 12297646, upload-time = "2026-07-12T20:21:45.76Z" }, + { url = "https://files.pythonhosted.org/packages/1d/37/2e9c94f0b383d8cbe1a35517ab470b7810bc9d7501603ab532bcd5be5e90/ty-0.0.59-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fd53b8581641d8dad7bfac6d5ea589e91a883d6837e0b9a286fdae30722b7c69", size = 11432519, upload-time = "2026-07-12T20:21:47.694Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0a/af93e9785200f11ac416cc20235fc2464c9bd978e791190684ea0e458795/ty-0.0.59-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:86da5872124a41877d95058bc17d33ddcff034b587eb5f1e2917ab88ba227dac", size = 11554993, upload-time = "2026-07-12T20:21:49.671Z" }, + { url = "https://files.pythonhosted.org/packages/4b/dd/651bf87e20d00376c81b19124756491cffaf20eb8bec05a8794e5a8cf641/ty-0.0.59-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6a233eef5f2fd4d894881e4a0aec83c9f172bfae1d787d6596ee1939fcc7723e", size = 11818230, upload-time = "2026-07-12T20:21:51.659Z" }, + { url = "https://files.pythonhosted.org/packages/16/50/c947c4155fea751d135b19affdf734bbce72a94e446b866cf0c62f8bed69/ty-0.0.59-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7ff678c18b5f1e3128b75a35e50dee7908dea55155baa31cd790619d5014cbf5", size = 12135194, upload-time = "2026-07-12T20:21:53.796Z" }, + { url = "https://files.pythonhosted.org/packages/b1/13/e5feb138888de1e95037c843571bbbd4ac21bf0a190507468098599a321f/ty-0.0.59-py3-none-win32.whl", hash = "sha256:cf8abb4b8095c5fe39102b8127f5886db308c8d4600909ddbc905512ce9c8163", size = 11179249, upload-time = "2026-07-12T20:21:55.752Z" }, + { url = "https://files.pythonhosted.org/packages/76/dd/52914dcbeeba92c207de40ef7109a58dcb5527aeb21c8f8feb7402aa9e29/ty-0.0.59-py3-none-win_amd64.whl", hash = "sha256:1dde20a82243d24407869e5a608c2f15efddd5cefc662aef461a5af84bfb3f8b", size = 12251079, upload-time = "2026-07-12T20:21:58.1Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8f/ac36fde77e223297454c1e0aeb8888c169eaacf3163bb609e3af942c88cb/ty-0.0.59-py3-none-win_arm64.whl", hash = "sha256:987043ee9e021f49493d9135891ac69c1affeee0d4ad4480c5fa4d9c975fc91b", size = 11650921, upload-time = "2026-07-12T20:22:00.348Z" }, +] + [[package]] name = "typer" version = "0.26.7"