Skip to content

Commit 50e05bf

Browse files
fix: address P0-3 demo seed remaining blockers
Dashboard: - Add row-to-object transformation in query() function to fix undefined/NaN values - API returns {columns: [...], rows: [[...]]} but JS expected {col: val} objects pg_stat_statements in Brain: - Add explicit exclusion filter (EXCLUDED_EXTENSION_VIEWS_SQL) in PostgresIntrospectionProvider - Excludes pg_stat_statements, pg_stat_statements_info, pg_buffercache by name - Belt-and-suspenders approach alongside existing pg_catalog schema filter - Create extension in 'extensions' schema for best practices Digest: - Improve seed script to verify digest creation after trigger - Wait longer (5s) for async generation to complete - Only use deterministic fallback if no digest exists - Fallback includes slow query count alongside index recommendations Testing: - Add automated tests for pg_stat_statements exclusion predicates - Verify both schema exclusion and extension view name exclusion work Co-authored-by: Venkat SF <venkatesh.sakamuri@stayflexi.com>
1 parent 976bc48 commit 50e05bf

5 files changed

Lines changed: 200 additions & 107 deletions

File tree

‎backend/src/main/java/com/dbaagent/provider/postgres/PostgresIntrospectionProvider.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,22 @@ public class PostgresIntrospectionProvider implements IntrospectionProvider {
2929
+ "AND %1$s NOT LIKE 'pg_temp_%%' "
3030
+ "AND %1$s NOT LIKE 'pg_toast_temp_%%'";
3131

32+
/**
33+
* Extension-created system views that should be excluded from Brain even if
34+
* they somehow appear in a user schema. pg_stat_statements is in pg_catalog
35+
* (so already excluded by schema filter), but this provides defense in depth.
36+
*/
37+
static final String EXCLUDED_EXTENSION_VIEWS_SQL =
38+
"NOT IN ('pg_stat_statements', 'pg_stat_statements_info', 'pg_buffercache')";
39+
3240
static String nonSystemSchemaPredicate(String column) {
3341
return column + " " + String.format(NON_SYSTEM_SCHEMA_SQL, column);
3442
}
3543

44+
static String excludeExtensionViewsPredicate(String column) {
45+
return column + " " + EXCLUDED_EXTENSION_VIEWS_SQL;
46+
}
47+
3648
/** Map / snapshot key that survives duplicate table names across schemas. */
3749
static String qualifiedTableKey(String schema, String table) {
3850
String s = (schema == null || schema.isBlank()) ? DEFAULT_SCHEMA : schema;
@@ -74,6 +86,7 @@ private List<DatabaseObject> getTablesAndViews(Connection connection) throws SQL
7486

7587
String schemaPred = nonSystemSchemaPredicate("t.schemaname");
7688
String viewPred = nonSystemSchemaPredicate("v.schemaname");
89+
String extViewPred = excludeExtensionViewsPredicate("v.viewname");
7790
String query = """
7891
SELECT t.schemaname as schema_name, t.tablename as name, 'table' as type,
7992
CASE
@@ -89,9 +102,9 @@ private List<DatabaseObject> getTablesAndViews(Connection connection) throws SQL
89102
WHERE %s AND c.relkind IN ('r', 'p')
90103
UNION ALL
91104
SELECT v.schemaname as schema_name, v.viewname as name, 'view' as type, 0 as row_count
92-
FROM pg_views v WHERE %s
105+
FROM pg_views v WHERE %s AND %s
93106
ORDER BY schema_name, type, name
94-
""".formatted(schemaPred, viewPred);
107+
""".formatted(schemaPred, viewPred, extViewPred);
95108

96109
try (Statement stmt = connection.createStatement();
97110
ResultSet rs = stmt.executeQuery(query)) {

‎backend/src/test/java/com/dbaagent/provider/postgres/PostgresIntrospectionProviderTest.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,4 +339,77 @@ void getColumnDetails_returnsDetails() throws SQLException {
339339
assertEquals("character varying", details.get(0).getDataType());
340340
assertFalse(details.get(0).getIsNullable());
341341
}
342+
343+
// ─── pg_stat_statements extension view exclusion tests ───────────────────────
344+
345+
@Test
346+
void excludeExtensionViewsPredicate_excludesPgStatStatements() {
347+
String predicate = PostgresIntrospectionProvider.excludeExtensionViewsPredicate("v.viewname");
348+
349+
assertTrue(predicate.contains("pg_stat_statements"),
350+
"Exclusion predicate should mention pg_stat_statements");
351+
assertTrue(predicate.contains("pg_stat_statements_info"),
352+
"Exclusion predicate should mention pg_stat_statements_info");
353+
assertTrue(predicate.contains("pg_buffercache"),
354+
"Exclusion predicate should mention pg_buffercache");
355+
assertTrue(predicate.contains("NOT IN"),
356+
"Exclusion predicate should use NOT IN clause");
357+
}
358+
359+
@Test
360+
void nonSystemSchemaPredicate_excludesPgCatalog() {
361+
String predicate = PostgresIntrospectionProvider.nonSystemSchemaPredicate("t.schemaname");
362+
363+
assertTrue(predicate.contains("pg_catalog"),
364+
"Schema predicate should exclude pg_catalog");
365+
assertTrue(predicate.contains("information_schema"),
366+
"Schema predicate should exclude information_schema");
367+
assertTrue(predicate.contains("pg_toast"),
368+
"Schema predicate should exclude pg_toast");
369+
}
370+
371+
@Test
372+
void getTablesAndViews_queryExcludesExtensionViews() throws SQLException {
373+
// Capture the SQL query and verify it contains the extension view exclusion
374+
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
375+
when(connection.createStatement()).thenReturn(statement);
376+
when(statement.executeQuery(sqlCaptor.capture())).thenReturn(resultSet);
377+
when(resultSet.next()).thenReturn(false);
378+
379+
// getDatabaseObjects calls getTablesAndViews internally
380+
List<DatabaseObject> objects = provider.getDatabaseObjects(connection, "test_db");
381+
382+
// Verify the query was captured
383+
String executedQuery = sqlCaptor.getValue();
384+
assertNotNull(executedQuery);
385+
386+
// The query for views should contain the extension exclusion
387+
// (tables + views are in one UNION query, so check the whole thing)
388+
assertTrue(executedQuery.contains("pg_views") || executedQuery.contains("pg_tables"),
389+
"Query should access PostgreSQL system catalogs");
390+
391+
// Verify schema exclusions are present
392+
assertTrue(executedQuery.contains("NOT IN"),
393+
"Query should have NOT IN clause for exclusions");
394+
assertTrue(executedQuery.contains("pg_catalog") || executedQuery.contains("pg_stat_statements"),
395+
"Query should exclude system schemas or extension views");
396+
}
397+
398+
@Test
399+
void extensionViewExclusion_isExactMatch() {
400+
// Verify the exclusion predicate uses exact matches, not prefix matches
401+
String predicate = PostgresIntrospectionProvider.EXCLUDED_EXTENSION_VIEWS_SQL;
402+
403+
// The excluded list should be specific, exact names only
404+
assertTrue(predicate.contains("'pg_stat_statements'"),
405+
"Predicate should exclude exactly 'pg_stat_statements'");
406+
assertTrue(predicate.contains("'pg_stat_statements_info'"),
407+
"Predicate should exclude exactly 'pg_stat_statements_info'");
408+
assertTrue(predicate.contains("'pg_buffercache'"),
409+
"Predicate should exclude exactly 'pg_buffercache'");
410+
411+
// Verify it's a NOT IN list (exact match semantics, not LIKE pattern)
412+
assertTrue(predicate.startsWith("NOT IN"),
413+
"Predicate should use NOT IN for exact matching");
414+
}
342415
}

‎docker/postgres/init/10_create_demo_shop.sql‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,13 @@ CREATE DATABASE demo_shop;
1111

1212
\connect demo_shop
1313

14-
-- Enable extensions
15-
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
14+
-- Create separate schema for extensions to keep them out of Brain indexing
15+
CREATE SCHEMA IF NOT EXISTS extensions;
16+
17+
-- Enable extensions in the extensions schema (keeps pg_stat_statements out of public schema)
18+
-- Note: pg_stat_statements doesn't actually create tables in any schema, but this is good practice
19+
-- The extension's view is system-wide and accessed via pg_catalog, not the target schema
20+
CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions;
1621

1722
-- ============================================================================
1823
-- SCHEMA: Core E-commerce Tables

‎scripts/self-host/demo-dashboard.html‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,18 @@ <h3>🏆 Top 10 Products by Revenue</h3>
4848
</div>
4949
<script>
5050
async function query(sql) {
51-
return deepsql.query(sql);
51+
const result = await deepsql.query(sql);
52+
// Transform array rows to objects using column names
53+
// API returns: {columns: ["col1", "col2"], rows: [[val1, val2], ...]}
54+
// We need: [{col1: val1, col2: val2}, ...]
55+
if (result.columns && result.rows) {
56+
result.rows = result.rows.map(row => {
57+
const obj = {};
58+
result.columns.forEach((col, i) => obj[col] = row[i]);
59+
return obj;
60+
});
61+
}
62+
return result;
5263
}
5364
async function loadDashboard() {
5465
try {

0 commit comments

Comments
 (0)