diff --git a/README.md b/README.md index 0d51396..de6224a 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,62 @@ which parses/introspects/diffs like any other column type; see handled on the CRUD side (write-side serialization only, no auto-parsing on read — `mssql-python` doesn't distinguish `json` columns from `nvarchar`). +### Area tagging and filtering + +Any migration file or `database/` code file can declare one or more areas by +starting with a `-- area:` comment: + +```sql +-- area: billing +CREATE TABLE billing.invoices (id int primary key); +``` + +A file can declare more than one area, either comma-separated on one line +(`-- area: billing, reporting`) or across several `-- area:` lines — the +declared areas union. The directive is only recognized in the file's leading +comment block (blank lines and `--` comments at the very top, stopping at the +first real statement); a `-- area:` comment later in the file doesn't count. +A file with no directive is untagged, and untagged files are treated as +shared/common. + +`pgdb compare`, `pgdb migrate check`, and `pgdb migrate apply` all accept: + +- `--area NAME` (repeatable) — restrict to files declaring one of the given + areas, **plus every untagged file** (untagged files always stay in scope). +- `--exclude-area NAME` (repeatable) — drop files declaring one of the given + areas; untagged files are never dropped by this. + +Both can be combined; a file matching both an included and an excluded area +is excluded. Passing neither option applies no filtering (the default, +unchanged behavior). + +```bash +pgdb migrate apply path/to/database/_migration_scripts --url ... --area billing +pgdb compare path/to/database/ --url ... --exclude-area reporting +``` + +`compare`'s default report (no `--report-extra-db`) only checks that the +filtered scripts exist correctly in the DB, so it composes safely with area +filtering. Passing `--report-extra-db` together with an area filter also +reports every DB object outside the filtered area(s) as "missing in +scripts" — since the live database has no concept of areas, only the +scripts side is filtered — so treat that combination's "missing in scripts" +results with that in mind (the CLI prints a warning when you combine them). + +`pgdb fetch-missing` deliberately has **no** `--area`/`--exclude-area`: it +diffs the full database against scripts to find genuinely untracked +objects, so narrowing the scripts side by area would make every object +tracked only under a different area look "missing" too — and `--write` +would then reconstruct a duplicate file for something that already exists. + +`pgdevkit.areas` exposes the same logic for scripting: +`parse_areas`/`file_areas` read a file's declared areas, and +`area_allowed`/`filter_by_area` apply the `only`/`exclude` semantics above. +`pgdevkit.migrate.list_migration_files`/`pending_migrations` and +`pgdevkit.parser.parse_directory` take the same `areas`/`exclude_areas` +keyword arguments (`pgdevkit.fetch_missing.find_missing_objects` doesn't, +for the reason above). + ## `pgdb testdb` Manages a single shared, Podman-backed Postgres container for local tests diff --git a/pgdevkit/areas.py b/pgdevkit/areas.py new file mode 100644 index 0000000..a8c28d2 --- /dev/null +++ b/pgdevkit/areas.py @@ -0,0 +1,64 @@ +"""Optional `-- area: NAME[, NAME...]` tag recognized in the leading comment +block of a migration file or a `database/` code file (blank lines and `--` +comments at the very top, stopping at the first real statement — like a file +header). A file may declare more than one area, either as a comma-separated +list on one line or across several `-- area:` lines (the areas union). + +A file with no such directive is "untagged" and is treated as shared/common: +`only` filters always keep untagged files, and `exclude` filters never drop +them — only a file that explicitly declares an excluded area is dropped. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +_AREA_LINE = re.compile(r"^\s*--\s*area\s*:\s*(.+?)\s*$", re.IGNORECASE) + + +def parse_areas(content: str) -> frozenset[str]: + """Area names declared in `content`'s leading comment block.""" + areas: set[str] = set() + for line in content.splitlines(): + stripped = line.strip() + if not stripped: + continue + if not stripped.startswith("--"): + break + m = _AREA_LINE.match(stripped) + if m: + areas.update(a.strip() for a in m.group(1).split(",") if a.strip()) + return frozenset(areas) + + +def file_areas(path: Path) -> frozenset[str]: + """Area names declared in the file at `path`.""" + return parse_areas(path.read_text(encoding="utf-8")) + + +def area_allowed( + areas: frozenset[str], + *, + only: frozenset[str] | None = None, + exclude: frozenset[str] | None = None, +) -> bool: + """Whether a file that declares `areas` passes an `only`/`exclude` filter.""" + if exclude and areas & exclude: + return False + if only and areas and not (areas & only): + return False + return True + + +def filter_by_area( + paths: list[Path], + *, + only: frozenset[str] | None = None, + exclude: frozenset[str] | None = None, +) -> list[Path]: + """`paths` restricted by an `only`/`exclude` area filter. Returns `paths` + unchanged (no file reads) when neither filter is set.""" + if not only and not exclude: + return paths + return [p for p in paths if area_allowed(file_areas(p), only=only, exclude=exclude)] diff --git a/pgdevkit/cli.py b/pgdevkit/cli.py index f8968b7..9cdacdf 100644 --- a/pgdevkit/cli.py +++ b/pgdevkit/cli.py @@ -23,6 +23,17 @@ console = Console() err_console = Console(stderr=True) +_AREA_OPTION = typer.Option( + [], "--area", help="Restrict to files declaring this area (repeatable); untagged files always stay in scope" +) +_EXCLUDE_AREA_OPTION = typer.Option( + [], "--exclude-area", help="Skip files declaring this area (repeatable); untagged files are never excluded" +) + + +def _as_area_set(values: list[str]) -> frozenset[str] | None: + return frozenset(values) if values else None + testdb_app = typer.Typer(name="testdb", help="Manage the shared local Postgres test container") app.add_typer(testdb_app, name="testdb") @@ -46,12 +57,20 @@ def compare( ), report_extra_db: bool = typer.Option(False, "--report-extra-db", help="Report objects in DB but not in scripts"), dialect: str = typer.Option("postgres", "--dialect", help="postgres (default) or mssql"), + area: list[str] = _AREA_OPTION, + exclude_area: list[str] = _EXCLUDE_AREA_OPTION, scripts_dir: Path = typer.Argument(..., help="Directory containing SQL scripts"), ) -> None: """Compare SQL scripts to a live database and report differences.""" if not scripts_dir.is_dir(): err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory") raise typer.Exit(2) + if report_extra_db and (area or exclude_area): + console.print( + "[yellow]⚠[/yellow] --report-extra-db with --area/--exclude-area will report every DB object " + "outside the filtered area(s) as \"missing in scripts\", since the live database has no concept " + "of areas — only the scripts side is filtered." + ) try: conninfo = build_conninfo( @@ -71,7 +90,9 @@ def compare( raise typer.Exit(2) with console.status("Parsing SQL scripts..."): - scripts_schema = parse_directory(scripts_dir, dialect=backend.dialect) + scripts_schema = parse_directory( + scripts_dir, dialect=backend.dialect, areas=_as_area_set(area), exclude_areas=_as_area_set(exclude_area) + ) with console.status("Introspecting database..."): db_schema = backend.introspect(conninfo) @@ -110,7 +131,11 @@ def fetch_missing( only: list[str] = typer.Option([], "--only", help="Only fetch schema.name (repeatable); default is everything"), ) -> None: """Find tables/views/functions that exist in the database but aren't - tracked under scripts_dir, and reverse-engineer their DDL into new files.""" + tracked under scripts_dir, and reverse-engineer their DDL into new files. + + No --area/--exclude-area here (unlike compare/migrate): this diffs the + full live database against scripts, so filtering the scripts side by + area would misreport objects tracked under other areas as missing.""" if not scripts_dir.is_dir(): err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory") raise typer.Exit(2) @@ -245,6 +270,8 @@ def migrate_check( help="schema.table recording applied migrations " "(default: tool.pgdevkit.migrations_table in pyproject.toml, else public.schema_migrations)", ), + area: list[str] = _AREA_OPTION, + exclude_area: list[str] = _EXCLUDE_AREA_OPTION, ) -> None: """List which migration files under migrations_dir are applied vs. pending.""" if not migrations_dir.is_dir(): @@ -253,7 +280,9 @@ def migrate_check( conninfo = build_conninfo(url, entra_user) tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir) - local_files = migrate.list_migration_files(migrations_dir) + local_files = migrate.list_migration_files( + migrations_dir, areas=_as_area_set(area), exclude_areas=_as_area_set(exclude_area) + ) try: applied = migrate.applied_migrations(conninfo, tracking_table) except migrate.TrackingTableMissing: @@ -292,6 +321,8 @@ def migrate_apply( ), ask: bool = typer.Option(False, "--ask", help="Show and confirm each migration before running it"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm-target prompt"), + area: list[str] = _AREA_OPTION, + exclude_area: list[str] = _EXCLUDE_AREA_OPTION, ) -> None: """Apply pending migration files, in filename order, tracking each in tracking_table.""" if not migrations_dir.is_dir(): @@ -300,6 +331,7 @@ def migrate_apply( conninfo = build_conninfo(url, entra_user) tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir) + areas, exclude_areas = _as_area_set(area), _as_area_set(exclude_area) target_desc = url.rsplit("@", 1)[-1] if "@" in url else url if not yes: typer.confirm(f"About to run migrations against {target_desc}. Continue?", abort=True) @@ -308,12 +340,14 @@ def migrate_apply( targets = [migrations_dir / file] else: try: - targets = migrate.pending_migrations(migrations_dir, conninfo, tracking_table) + targets = migrate.pending_migrations( + migrations_dir, conninfo, tracking_table, areas=areas, exclude_areas=exclude_areas + ) except migrate.TrackingTableMissing: err_console.print( f"[yellow]⚠[/yellow] {tracking_table} not found — treating every migration as pending" ) - targets = migrate.list_migration_files(migrations_dir) + targets = migrate.list_migration_files(migrations_dir, areas=areas, exclude_areas=exclude_areas) if not targets: console.print("No pending migrations.") diff --git a/pgdevkit/fetch_missing.py b/pgdevkit/fetch_missing.py index 9d014f3..41edaf5 100644 --- a/pgdevkit/fetch_missing.py +++ b/pgdevkit/fetch_missing.py @@ -50,7 +50,12 @@ def layer_folder_for(scripts_dir: Path, schema: str) -> Path: def find_missing_objects(scripts_dir: Path, conninfo: str) -> list[MissingObject]: """Tables, views, and functions that exist in the live database but - aren't tracked as .sql files under scripts_dir.""" + aren't tracked as .sql files under scripts_dir. + + Deliberately not area-filterable: this diffs the *full* database against + scripts with report_extra_db=True, so narrowing the scripts side to one + area would make every object tracked only under a different area look + "missing" too — and --write would reconstruct a duplicate file for it.""" scripts = parse_directory(scripts_dir) db = introspect_db(conninfo) diffs = compute_diff(scripts, db, report_extra_db=True) diff --git a/pgdevkit/migrate.py b/pgdevkit/migrate.py index 5cd377d..ad0289e 100644 --- a/pgdevkit/migrate.py +++ b/pgdevkit/migrate.py @@ -18,6 +18,8 @@ from psycopg import errors as pg_errors from psycopg import sql as pg_sql +from .areas import filter_by_area + _IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*" _DEFAULT_TRACKING_TABLE = "public.schema_migrations" @@ -274,8 +276,14 @@ def already_fully_applied(conninfo: str, path: Path) -> bool: return all(_target_exists(con, cast(tuple[str, ...], t)) for t in targets) -def list_migration_files(migrations_dir: Path) -> list[Path]: - return sorted(migrations_dir.glob("*.sql")) +def list_migration_files( + migrations_dir: Path, + *, + areas: frozenset[str] | None = None, + exclude_areas: frozenset[str] | None = None, +) -> list[Path]: + files = sorted(migrations_dir.glob("*.sql")) + return filter_by_area(files, only=areas, exclude=exclude_areas) def applied_migrations(conninfo: str, tracking_table: str) -> dict[str, tuple[datetime, str]]: @@ -291,9 +299,17 @@ def applied_migrations(conninfo: str, tracking_table: str) -> dict[str, tuple[da return {r[0]: (r[1], r[2]) for r in rows} -def pending_migrations(migrations_dir: Path, conninfo: str, tracking_table: str) -> list[Path]: +def pending_migrations( + migrations_dir: Path, + conninfo: str, + tracking_table: str, + *, + areas: frozenset[str] | None = None, + exclude_areas: frozenset[str] | None = None, +) -> list[Path]: applied = applied_migrations(conninfo, tracking_table) - return [p for p in list_migration_files(migrations_dir) if p.name not in applied] + files = list_migration_files(migrations_dir, areas=areas, exclude_areas=exclude_areas) + return [p for p in files if p.name not in applied] def record_applied(conninfo: str, tracking_table: str, filename: str) -> bool: diff --git a/pgdevkit/parser.py b/pgdevkit/parser.py index 06226c9..c89c6d5 100644 --- a/pgdevkit/parser.py +++ b/pgdevkit/parser.py @@ -8,6 +8,7 @@ import sqlglot import sqlglot.expressions as exp +from .areas import area_allowed, parse_areas from .dialect import Dialect, POSTGRES, resolve_dialect from .models import ( ColumnDef, ConstraintDef, CompositeTypeDef, DatabaseSchema, @@ -48,16 +49,30 @@ def _iter_sql_files(scripts_dir: Path): yield Path(root) / name -def parse_directory(scripts_dir: Path, *, dialect: str | Dialect = "postgres") -> DatabaseSchema: +def parse_directory( + scripts_dir: Path, + *, + dialect: str | Dialect = "postgres", + areas: frozenset[str] | None = None, + exclude_areas: frozenset[str] | None = None, +) -> DatabaseSchema: resolved = resolve_dialect(dialect) db_schema = DatabaseSchema() for sql_file in sorted(_iter_sql_files(scripts_dir)): - _parse_file(sql_file, db_schema, resolved) + # Read once and reuse for both the area check and parsing, rather than + # filtering the file list up front (which would need its own read). + content = sql_file.read_text(encoding="utf-8") + if (areas or exclude_areas) and not area_allowed(parse_areas(content), only=areas, exclude=exclude_areas): + continue + _parse_file(sql_file, db_schema, resolved, content=content) return db_schema -def _parse_file(path: Path, db_schema: DatabaseSchema, dialect: Dialect = POSTGRES) -> None: - content = path.read_text(encoding="utf-8") +def _parse_file( + path: Path, db_schema: DatabaseSchema, dialect: Dialect = POSTGRES, content: str | None = None +) -> None: + if content is None: + content = path.read_text(encoding="utf-8") try: exprs = sqlglot.parse(content, dialect=dialect.sqlglot_name, error_level=sqlglot.ErrorLevel.WARN) except Exception as e: diff --git a/pyproject.toml b/pyproject.toml index 781b0fb..14d1801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ packages = ["pgdevkit"] [project] name = "pgdevkit" -version = "0.3.8" +version = "0.4.0" description = "A helper for developing with Postgres" readme = "README.md" requires-python = ">=3.14" diff --git a/tests/test_areas.py b/tests/test_areas.py new file mode 100644 index 0000000..e7fab46 --- /dev/null +++ b/tests/test_areas.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from pathlib import Path + +from pgdevkit.areas import area_allowed, file_areas, filter_by_area, parse_areas + + +class TestParseAreas: + def test_no_directive_is_untagged(self): + assert parse_areas("CREATE TABLE t (id int);\n") == frozenset() + + def test_single_area(self): + sql = "-- area: billing\nCREATE TABLE t (id int);\n" + assert parse_areas(sql) == frozenset({"billing"}) + + def test_comma_separated_areas(self): + sql = "-- area: billing, reporting\nCREATE TABLE t (id int);\n" + assert parse_areas(sql) == frozenset({"billing", "reporting"}) + + def test_repeated_lines_union(self): + sql = "-- area: billing\n-- area: reporting\nCREATE TABLE t (id int);\n" + assert parse_areas(sql) == frozenset({"billing", "reporting"}) + + def test_case_insensitive_directive(self): + sql = "-- Area: billing\nCREATE TABLE t (id int);\n" + assert parse_areas(sql) == frozenset({"billing"}) + + def test_blank_lines_before_directive_are_skipped(self): + sql = "\n\n-- area: billing\nCREATE TABLE t (id int);\n" + assert parse_areas(sql) == frozenset({"billing"}) + + def test_directive_after_other_leading_comments_still_counts(self): + sql = "-- Copyright 2026\n-- area: billing\nCREATE TABLE t (id int);\n" + assert parse_areas(sql) == frozenset({"billing"}) + + def test_directive_past_leading_comment_block_is_ignored(self): + sql = "CREATE TABLE t (id int);\n-- area: billing\n" + assert parse_areas(sql) == frozenset() + + def test_directive_stops_at_first_blank_then_statement(self): + # A directive after a blank line inside the header still counts (blank + # lines don't end the header), but nothing after the first real + # statement does, even inside a later comment. + sql = "-- area: billing\n\nCREATE TABLE t (id int);\n-- area: reporting\n" + assert parse_areas(sql) == frozenset({"billing"}) + + +class TestFileAreas: + def test_reads_from_disk(self, tmp_path: Path): + f = tmp_path / "001_thing.sql" + f.write_text("-- area: billing\nCREATE TABLE t (id int);\n", encoding="utf-8") + assert file_areas(f) == frozenset({"billing"}) + + +class TestAreaAllowed: + def test_no_filters_always_allowed(self): + assert area_allowed(frozenset({"billing"})) is True + assert area_allowed(frozenset()) is True + + def test_only_filter_untagged_always_passes(self): + assert area_allowed(frozenset(), only=frozenset({"billing"})) is True + + def test_only_filter_matching_area_passes(self): + assert area_allowed(frozenset({"billing"}), only=frozenset({"billing"})) is True + + def test_only_filter_non_matching_area_fails(self): + assert area_allowed(frozenset({"reporting"}), only=frozenset({"billing"})) is False + + def test_exclude_filter_untagged_never_dropped(self): + assert area_allowed(frozenset(), exclude=frozenset({"billing"})) is True + + def test_exclude_filter_matching_area_dropped(self): + assert area_allowed(frozenset({"billing"}), exclude=frozenset({"billing"})) is False + + def test_exclude_filter_non_matching_area_passes(self): + assert area_allowed(frozenset({"reporting"}), exclude=frozenset({"billing"})) is True + + def test_only_and_exclude_combined_exclude_wins(self): + areas = frozenset({"billing"}) + assert area_allowed(areas, only=frozenset({"billing"}), exclude=frozenset({"billing"})) is False + + +class TestFilterByArea: + def test_no_filters_returns_paths_unchanged(self, tmp_path: Path): + paths = [tmp_path / "a.sql", tmp_path / "b.sql"] + assert filter_by_area(paths) == paths + + def test_only_keeps_matching_and_untagged(self, tmp_path: Path): + tagged = tmp_path / "billing.sql" + tagged.write_text("-- area: billing\nselect 1;\n", encoding="utf-8") + other = tmp_path / "reporting.sql" + other.write_text("-- area: reporting\nselect 1;\n", encoding="utf-8") + untagged = tmp_path / "common.sql" + untagged.write_text("select 1;\n", encoding="utf-8") + + result = filter_by_area([tagged, other, untagged], only=frozenset({"billing"})) + assert set(result) == {tagged, untagged} + + def test_exclude_drops_matching_but_keeps_untagged(self, tmp_path: Path): + tagged = tmp_path / "billing.sql" + tagged.write_text("-- area: billing\nselect 1;\n", encoding="utf-8") + untagged = tmp_path / "common.sql" + untagged.write_text("select 1;\n", encoding="utf-8") + + result = filter_by_area([tagged, untagged], exclude=frozenset({"billing"})) + assert result == [untagged] diff --git a/tests/test_migrate_areas.py b/tests/test_migrate_areas.py new file mode 100644 index 0000000..80eccdb --- /dev/null +++ b/tests/test_migrate_areas.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pathlib import Path + +from pgdevkit.migrate import list_migration_files + + +def _write(dir: Path, name: str, content: str) -> Path: + p = dir / name + p.write_text(content, encoding="utf-8") + return p + + +class TestListMigrationFilesAreaFiltering: + def test_no_filters_lists_everything(self, tmp_path: Path): + a = _write(tmp_path, "001_a.sql", "-- area: billing\nselect 1;\n") + b = _write(tmp_path, "002_b.sql", "select 1;\n") + assert list_migration_files(tmp_path) == sorted([a, b]) + + def test_areas_filter_keeps_matching_and_untagged(self, tmp_path: Path): + billing = _write(tmp_path, "001_billing.sql", "-- area: billing\nselect 1;\n") + reporting = _write(tmp_path, "002_reporting.sql", "-- area: reporting\nselect 1;\n") + common = _write(tmp_path, "003_common.sql", "select 1;\n") + + result = list_migration_files(tmp_path, areas=frozenset({"billing"})) + assert set(result) == {billing, common} + assert reporting not in result + + def test_exclude_areas_drops_matching_but_keeps_untagged(self, tmp_path: Path): + billing = _write(tmp_path, "001_billing.sql", "-- area: billing\nselect 1;\n") + common = _write(tmp_path, "002_common.sql", "select 1;\n") + + result = list_migration_files(tmp_path, exclude_areas=frozenset({"billing"})) + assert result == [common] + assert billing not in result + + def test_multiple_areas_on_one_file(self, tmp_path: Path): + multi = _write(tmp_path, "001_multi.sql", "-- area: billing, reporting\nselect 1;\n") + + assert list_migration_files(tmp_path, areas=frozenset({"reporting"})) == [multi] + assert list_migration_files(tmp_path, exclude_areas=frozenset({"billing"})) == [] diff --git a/tests/test_parser_areas.py b/tests/test_parser_areas.py new file mode 100644 index 0000000..c8dffdd --- /dev/null +++ b/tests/test_parser_areas.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pathlib import Path + +from pgdevkit.parser import parse_directory + + +def _write(dir: Path, name: str, content: str) -> Path: + p = dir / name + p.write_text(content, encoding="utf-8") + return p + + +class TestParseDirectoryAreaFiltering: + def test_no_filters_parses_everything(self, tmp_path: Path): + _write(tmp_path, "billing.sql", "-- area: billing\nCREATE TABLE public.a (id int);\n") + _write(tmp_path, "common.sql", "CREATE TABLE public.b (id int);\n") + schema = parse_directory(tmp_path) + assert set(schema.tables) == {"public.a", "public.b"} + + def test_areas_filter_keeps_matching_and_untagged(self, tmp_path: Path): + _write(tmp_path, "billing.sql", "-- area: billing\nCREATE TABLE public.a (id int);\n") + _write(tmp_path, "reporting.sql", "-- area: reporting\nCREATE TABLE public.b (id int);\n") + _write(tmp_path, "common.sql", "CREATE TABLE public.c (id int);\n") + + schema = parse_directory(tmp_path, areas=frozenset({"billing"})) + assert set(schema.tables) == {"public.a", "public.c"} + + def test_exclude_areas_drops_matching_but_keeps_untagged(self, tmp_path: Path): + _write(tmp_path, "billing.sql", "-- area: billing\nCREATE TABLE public.a (id int);\n") + _write(tmp_path, "common.sql", "CREATE TABLE public.c (id int);\n") + + schema = parse_directory(tmp_path, exclude_areas=frozenset({"billing"})) + assert set(schema.tables) == {"public.c"} diff --git a/uv.lock b/uv.lock index bfd6d4b..e9540a1 100644 --- a/uv.lock +++ b/uv.lock @@ -313,7 +313,7 @@ wheels = [ [[package]] name = "pgdevkit" -version = "0.3.8" +version = "0.4.0" source = { editable = "." } dependencies = [ { name = "docker" },