Skip to content

Commit f91a51d

Browse files
committed
fixed skip column casing issues as well
Signed-off-by: Anant <75747269+Anant-gif@users.noreply.github.com>
1 parent 3ce6bfa commit f91a51d

2 files changed

Lines changed: 68 additions & 2 deletions

File tree

sqlmesh/core/table_diff.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -353,11 +353,17 @@ def row_diff(
353353
self, temp_schema: t.Optional[str] = None, skip_grain_check: bool = False
354354
) -> RowDiff:
355355
if self._row_diff is None:
356+
source_skip_columns = {
357+
self._resolve_column_name(c, self.source_schema) for c in self.skip_columns
358+
}
359+
target_skip_columns = {
360+
self._resolve_column_name(c, self.target_schema) for c in self.skip_columns
361+
}
356362
source_schema = {
357-
c: t for c, t in self.source_schema.items() if c not in self.skip_columns
363+
c: t for c, t in self.source_schema.items() if c not in source_skip_columns
358364
}
359365
target_schema = {
360-
c: t for c, t in self.target_schema.items() if c not in self.skip_columns
366+
c: t for c, t in self.target_schema.items() if c not in target_skip_columns
361367
}
362368

363369
s_selects = {c: exp.column(c, "s").as_(f"s__{c}") for c in source_schema}

tests/core/test_table_diff.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1256,6 +1256,66 @@ def test_data_diff_non_lowercase_key_columns():
12561256
assert diff.t_only_count == 1
12571257

12581258

1259+
def test_data_diff_non_lowercase_skip_columns():
1260+
engine_adapter = DuckDBConnectionConfig().create_engine_adapter()
1261+
1262+
columns_to_types = {
1263+
"KEY": exp.DataType.build("int"),
1264+
"VALUE": exp.DataType.build("varchar"),
1265+
"IGNORED": exp.DataType.build("varchar"),
1266+
}
1267+
1268+
engine_adapter.create_table("src", columns_to_types)
1269+
engine_adapter.create_table("target", columns_to_types)
1270+
engine_adapter.insert_append(
1271+
"src",
1272+
pd.DataFrame([(1, "same", "source")], columns=columns_to_types),
1273+
)
1274+
engine_adapter.insert_append(
1275+
"target",
1276+
pd.DataFrame([(1, "same", "target")], columns=columns_to_types),
1277+
)
1278+
1279+
diff = TableDiff(
1280+
adapter=engine_adapter,
1281+
source="src",
1282+
target="target",
1283+
on=["key"],
1284+
skip_columns=["ignored"],
1285+
).row_diff()
1286+
1287+
assert diff.full_match_count == 1
1288+
assert diff.partial_match_count == 0
1289+
assert diff.column_stats.index.tolist() == ["VALUE"]
1290+
1291+
1292+
def test_skip_columns_across_schemas():
1293+
engine_adapter = DuckDBConnectionConfig().create_engine_adapter()
1294+
table_diff = TableDiff(
1295+
adapter=engine_adapter,
1296+
source="src",
1297+
target="target",
1298+
on=["key"],
1299+
skip_columns=["foo"],
1300+
)
1301+
data_type = exp.DataType.build("varchar")
1302+
table_diff.__dict__["source_schema"] = {"foo": data_type, "FOO": data_type}
1303+
table_diff.__dict__["target_schema"] = {"FOO": data_type}
1304+
1305+
source_skip_columns = {
1306+
table_diff._resolve_column_name(c, table_diff.source_schema)
1307+
for c in table_diff.skip_columns
1308+
}
1309+
target_skip_columns = {
1310+
table_diff._resolve_column_name(c, table_diff.target_schema)
1311+
for c in table_diff.skip_columns
1312+
}
1313+
1314+
assert source_skip_columns == {"foo"}
1315+
assert target_skip_columns == {"FOO"}
1316+
assert "FOO" not in source_skip_columns
1317+
1318+
12591319
def test_data_diff_key_columns_with_differing_case_between_source_and_target():
12601320
engine_adapter = DuckDBConnectionConfig().create_engine_adapter()
12611321

0 commit comments

Comments
 (0)