Address review: drop unsafe fetch-missing area filtering, avoid double file read

README.md

@@ -58,6 +58,62 @@ which parses/introspects/diffs like any other column type; see
 handled on the CRUD side (write-side serialization only, no auto-parsing on
 read — `mssql-python` doesn't distinguish `json` columns from `nvarchar`).
 
+### Area tagging and filtering
+
+Any migration file or `database/` code file can declare one or more areas by
+starting with a `-- area:` comment:
+
+```sql
+-- area: billing
+CREATE TABLE billing.invoices (id int primary key);
+```
+
+A file can declare more than one area, either comma-separated on one line
+(`-- area: billing, reporting`) or across several `-- area:` lines — the
+declared areas union. The directive is only recognized in the file's leading
+comment block (blank lines and `--` comments at the very top, stopping at the
+first real statement); a `-- area:` comment later in the file doesn't count.
+A file with no directive is untagged, and untagged files are treated as
+shared/common.
+
+`pgdb compare`, `pgdb migrate check`, and `pgdb migrate apply` all accept:
+
+- `--area NAME` (repeatable) — restrict to files declaring one of the given
+  areas, **plus every untagged file** (untagged files always stay in scope).
+- `--exclude-area NAME` (repeatable) — drop files declaring one of the given
+  areas; untagged files are never dropped by this.
+
+Both can be combined; a file matching both an included and an excluded area
+is excluded. Passing neither option applies no filtering (the default,
+unchanged behavior).
+
+```bash
+pgdb migrate apply path/to/database/_migration_scripts --url ... --area billing
+pgdb compare path/to/database/ --url ... --exclude-area reporting
+```
+
+`compare`'s default report (no `--report-extra-db`) only checks that the
+filtered scripts exist correctly in the DB, so it composes safely with area
+filtering. Passing `--report-extra-db` together with an area filter also
+reports every DB object outside the filtered area(s) as "missing in
+scripts" — since the live database has no concept of areas, only the
+scripts side is filtered — so treat that combination's "missing in scripts"
+results with that in mind (the CLI prints a warning when you combine them).
+
+`pgdb fetch-missing` deliberately has **no** `--area`/`--exclude-area`: it
+diffs the full database against scripts to find genuinely untracked
+objects, so narrowing the scripts side by area would make every object
+tracked only under a different area look "missing" too — and `--write`
+would then reconstruct a duplicate file for something that already exists.
+
+`pgdevkit.areas` exposes the same logic for scripting:
+`parse_areas`/`file_areas` read a file's declared areas, and
+`area_allowed`/`filter_by_area` apply the `only`/`exclude` semantics above.
+`pgdevkit.migrate.list_migration_files`/`pending_migrations` and
+`pgdevkit.parser.parse_directory` take the same `areas`/`exclude_areas`
+keyword arguments (`pgdevkit.fetch_missing.find_missing_objects` doesn't,
+for the reason above).
+
 ## `pgdb testdb`
 
 Manages a single shared, Podman-backed Postgres container for local tests

pgdevkit/areas.py

@@ -0,0 +1,64 @@
+"""Optional `-- area: NAME[, NAME...]` tag recognized in the leading comment
+block of a migration file or a `database/` code file (blank lines and `--`
+comments at the very top, stopping at the first real statement — like a file
+header). A file may declare more than one area, either as a comma-separated
+list on one line or across several `-- area:` lines (the areas union).
+
+A file with no such directive is "untagged" and is treated as shared/common:
+`only` filters always keep untagged files, and `exclude` filters never drop
+them — only a file that explicitly declares an excluded area is dropped.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+_AREA_LINE = re.compile(r"^\s*--\s*area\s*:\s*(.+?)\s*$", re.IGNORECASE)
+
+
+def parse_areas(content: str) -> frozenset[str]:
+    """Area names declared in `content`'s leading comment block."""
+    areas: set[str] = set()
+    for line in content.splitlines():
+        stripped = line.strip()
+        if not stripped:
+            continue
+        if not stripped.startswith("--"):
+            break
+        m = _AREA_LINE.match(stripped)
+        if m:
+            areas.update(a.strip() for a in m.group(1).split(",") if a.strip())
+    return frozenset(areas)
+
+
+def file_areas(path: Path) -> frozenset[str]:
+    """Area names declared in the file at `path`."""
+    return parse_areas(path.read_text(encoding="utf-8"))
+
+
+def area_allowed(
+    areas: frozenset[str],
+    *,
+    only: frozenset[str] | None = None,
+    exclude: frozenset[str] | None = None,
+) -> bool:
+    """Whether a file that declares `areas` passes an `only`/`exclude` filter."""
+    if exclude and areas & exclude:
+        return False
+    if only and areas and not (areas & only):
+        return False
+    return True
+
+
+def filter_by_area(
+    paths: list[Path],
+    *,
+    only: frozenset[str] | None = None,
+    exclude: frozenset[str] | None = None,
+) -> list[Path]:
+    """`paths` restricted by an `only`/`exclude` area filter. Returns `paths`
+    unchanged (no file reads) when neither filter is set."""
+    if not only and not exclude:
+        return paths
+    return [p for p in paths if area_allowed(file_areas(p), only=only, exclude=exclude)]

pgdevkit/cli.py

@@ -23,6 +23,17 @@
 console = Console()
 err_console = Console(stderr=True)
 
+_AREA_OPTION = typer.Option(
+    [], "--area", help="Restrict to files declaring this area (repeatable); untagged files always stay in scope"
+)
+_EXCLUDE_AREA_OPTION = typer.Option(
+    [], "--exclude-area", help="Skip files declaring this area (repeatable); untagged files are never excluded"
+)
+
+
+def _as_area_set(values: list[str]) -> frozenset[str] | None:
+    return frozenset(values) if values else None
+
 testdb_app = typer.Typer(name="testdb", help="Manage the shared local Postgres test container")
 app.add_typer(testdb_app, name="testdb")
 
@@ -46,12 +57,20 @@ def compare(
     ),
     report_extra_db: bool = typer.Option(False, "--report-extra-db", help="Report objects in DB but not in scripts"),
     dialect: str = typer.Option("postgres", "--dialect", help="postgres (default) or mssql"),
+    area: list[str] = _AREA_OPTION,
+    exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
     scripts_dir: Path = typer.Argument(..., help="Directory containing SQL scripts"),
 ) -> None:
     """Compare SQL scripts to a live database and report differences."""
     if not scripts_dir.is_dir():
         err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory")
         raise typer.Exit(2)
+    if report_extra_db and (area or exclude_area):
+        console.print(
+            "[yellow]⚠[/yellow]  --report-extra-db with --area/--exclude-area will report every DB object "
+            "outside the filtered area(s) as \"missing in scripts\", since the live database has no concept "
+            "of areas — only the scripts side is filtered."
+        )
 
     try:
         conninfo = build_conninfo(
@@ -71,7 +90,9 @@ def compare(
         raise typer.Exit(2)
 
     with console.status("Parsing SQL scripts..."):
-        scripts_schema = parse_directory(scripts_dir, dialect=backend.dialect)
+        scripts_schema = parse_directory(
+            scripts_dir, dialect=backend.dialect, areas=_as_area_set(area), exclude_areas=_as_area_set(exclude_area)
+        )
 
     with console.status("Introspecting database..."):
         db_schema = backend.introspect(conninfo)
@@ -110,7 +131,11 @@ def fetch_missing(
     only: list[str] = typer.Option([], "--only", help="Only fetch schema.name (repeatable); default is everything"),
 ) -> None:
     """Find tables/views/functions that exist in the database but aren't
-    tracked under scripts_dir, and reverse-engineer their DDL into new files."""
+    tracked under scripts_dir, and reverse-engineer their DDL into new files.
+
+    No --area/--exclude-area here (unlike compare/migrate): this diffs the
+    full live database against scripts, so filtering the scripts side by
+    area would misreport objects tracked under other areas as missing."""
     if not scripts_dir.is_dir():
         err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory")
         raise typer.Exit(2)
@@ -245,6 +270,8 @@ def migrate_check(
         help="schema.table recording applied migrations "
         "(default: tool.pgdevkit.migrations_table in pyproject.toml, else public.schema_migrations)",
     ),
+    area: list[str] = _AREA_OPTION,
+    exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
 ) -> None:
     """List which migration files under migrations_dir are applied vs. pending."""
     if not migrations_dir.is_dir():
@@ -253,7 +280,9 @@ def migrate_check(
 
     conninfo = build_conninfo(url, entra_user)
     tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir)
-    local_files = migrate.list_migration_files(migrations_dir)
+    local_files = migrate.list_migration_files(
+        migrations_dir, areas=_as_area_set(area), exclude_areas=_as_area_set(exclude_area)
+    )
     try:
         applied = migrate.applied_migrations(conninfo, tracking_table)
     except migrate.TrackingTableMissing:
@@ -292,6 +321,8 @@ def migrate_apply(
     ),
     ask: bool = typer.Option(False, "--ask", help="Show and confirm each migration before running it"),
     yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm-target prompt"),
+    area: list[str] = _AREA_OPTION,
+    exclude_area: list[str] = _EXCLUDE_AREA_OPTION,
 ) -> None:
     """Apply pending migration files, in filename order, tracking each in tracking_table."""
     if not migrations_dir.is_dir():
@@ -300,6 +331,7 @@ def migrate_apply(
 
     conninfo = build_conninfo(url, entra_user)
     tracking_table = tracking_table or migrate.default_tracking_table(migrations_dir)
+    areas, exclude_areas = _as_area_set(area), _as_area_set(exclude_area)
     target_desc = url.rsplit("@", 1)[-1] if "@" in url else url
     if not yes:
         typer.confirm(f"About to run migrations against {target_desc}. Continue?", abort=True)
@@ -308,12 +340,14 @@ def migrate_apply(
         targets = [migrations_dir / file]
     else:
         try:
-            targets = migrate.pending_migrations(migrations_dir, conninfo, tracking_table)
+            targets = migrate.pending_migrations(
+                migrations_dir, conninfo, tracking_table, areas=areas, exclude_areas=exclude_areas
+            )
         except migrate.TrackingTableMissing:
             err_console.print(
                 f"[yellow]⚠[/yellow]  {tracking_table} not found — treating every migration as pending"
             )
-            targets = migrate.list_migration_files(migrations_dir)
+            targets = migrate.list_migration_files(migrations_dir, areas=areas, exclude_areas=exclude_areas)
 
     if not targets:
         console.print("No pending migrations.")

pgdevkit/fetch_missing.py

@@ -50,7 +50,12 @@ def layer_folder_for(scripts_dir: Path, schema: str) -> Path:
 
 def find_missing_objects(scripts_dir: Path, conninfo: str) -> list[MissingObject]:
     """Tables, views, and functions that exist in the live database but
-    aren't tracked as .sql files under scripts_dir."""
+    aren't tracked as .sql files under scripts_dir.
+
+    Deliberately not area-filterable: this diffs the *full* database against
+    scripts with report_extra_db=True, so narrowing the scripts side to one
+    area would make every object tracked only under a different area look
+    "missing" too — and --write would reconstruct a duplicate file for it."""
     scripts = parse_directory(scripts_dir)
     db = introspect_db(conninfo)
     diffs = compute_diff(scripts, db, report_extra_db=True)

pgdevkit/migrate.py

@@ -18,6 +18,8 @@
 from psycopg import errors as pg_errors
 from psycopg import sql as pg_sql
 
+from .areas import filter_by_area
+
 _IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*"
 _DEFAULT_TRACKING_TABLE = "public.schema_migrations"
 
@@ -274,8 +276,14 @@ def already_fully_applied(conninfo: str, path: Path) -> bool:
         return all(_target_exists(con, cast(tuple[str, ...], t)) for t in targets)
 
 
-def list_migration_files(migrations_dir: Path) -> list[Path]:
-    return sorted(migrations_dir.glob("*.sql"))
+def list_migration_files(
+    migrations_dir: Path,
+    *,
+    areas: frozenset[str] | None = None,
+    exclude_areas: frozenset[str] | None = None,
+) -> list[Path]:
+    files = sorted(migrations_dir.glob("*.sql"))
+    return filter_by_area(files, only=areas, exclude=exclude_areas)
 
 
 def applied_migrations(conninfo: str, tracking_table: str) -> dict[str, tuple[datetime, str]]:
@@ -291,9 +299,17 @@ def applied_migrations(conninfo: str, tracking_table: str) -> dict[str, tuple[da
         return {r[0]: (r[1], r[2]) for r in rows}
 
 
-def pending_migrations(migrations_dir: Path, conninfo: str, tracking_table: str) -> list[Path]:
+def pending_migrations(
+    migrations_dir: Path,
+    conninfo: str,
+    tracking_table: str,
+    *,
+    areas: frozenset[str] | None = None,
+    exclude_areas: frozenset[str] | None = None,
+) -> list[Path]:
     applied = applied_migrations(conninfo, tracking_table)
-    return [p for p in list_migration_files(migrations_dir) if p.name not in applied]
+    files = list_migration_files(migrations_dir, areas=areas, exclude_areas=exclude_areas)
+    return [p for p in files if p.name not in applied]
 
 
 def record_applied(conninfo: str, tracking_table: str, filename: str) -> bool:

pgdevkit/parser.py

@@ -8,6 +8,7 @@
 import sqlglot
 import sqlglot.expressions as exp
 
+from .areas import area_allowed, parse_areas
 from .dialect import Dialect, POSTGRES, resolve_dialect
 from .models import (
     ColumnDef, ConstraintDef, CompositeTypeDef, DatabaseSchema,
@@ -48,16 +49,30 @@ def _iter_sql_files(scripts_dir: Path):
                 yield Path(root) / name
 
 
-def parse_directory(scripts_dir: Path, *, dialect: str | Dialect = "postgres") -> DatabaseSchema:
+def parse_directory(
+    scripts_dir: Path,
+    *,
+    dialect: str | Dialect = "postgres",
+    areas: frozenset[str] | None = None,
+    exclude_areas: frozenset[str] | None = None,
+) -> DatabaseSchema:
     resolved = resolve_dialect(dialect)
     db_schema = DatabaseSchema()
     for sql_file in sorted(_iter_sql_files(scripts_dir)):
-        _parse_file(sql_file, db_schema, resolved)
+        # Read once and reuse for both the area check and parsing, rather than
+        # filtering the file list up front (which would need its own read).
+        content = sql_file.read_text(encoding="utf-8")
+        if (areas or exclude_areas) and not area_allowed(parse_areas(content), only=areas, exclude=exclude_areas):
+            continue
+        _parse_file(sql_file, db_schema, resolved, content=content)
     return db_schema
 
 
-def _parse_file(path: Path, db_schema: DatabaseSchema, dialect: Dialect = POSTGRES) -> None:
-    content = path.read_text(encoding="utf-8")
+def _parse_file(
+    path: Path, db_schema: DatabaseSchema, dialect: Dialect = POSTGRES, content: str | None = None
+) -> None:
+    if content is None:
+        content = path.read_text(encoding="utf-8")
     try:
         exprs = sqlglot.parse(content, dialect=dialect.sqlglot_name, error_level=sqlglot.ErrorLevel.WARN)
     except Exception as e:

pyproject.toml

@@ -11,7 +11,7 @@ packages = ["pgdevkit"]
 
 [project]
 name = "pgdevkit"
-version = "0.3.8"
+version = "0.4.0"
 description = "A helper for developing with Postgres"
 readme = "README.md"
 requires-python = ">=3.14"