Skip to content

Commit ac7cc50

Browse files
cursoragentvenkateshsakamuri-lab
cursoragent
and
venkateshsakamuri-lab
committed
fix(seed): run workload queries as standalone SQL for pg_stat_statements
pg_stat_statements only tracks individual SQL statements, not: - Queries inside DO $$ PL/pgSQL blocks - LATERAL subqueries (tracked as part of outer query) Changed from LATERAL+generate_series approach to generating a SQL file with repeated individual SELECT statements that psql executes separately. Each statement is tracked by pg_stat_statements and aggregated under its queryid with cumulative calls and total_exec_time. This should populate the Performance tab with real slow queries from the 300K+ audit_log and 100K+ order_items tables. Co-authored-by: Venkat SF <venkatesh.sakamuri@stayflexi.com>
1 parent 03e6dd5 commit ac7cc50

1 file changed

Lines changed: 118 additions & 65 deletions

File tree

‎scripts/self-host/seed-demo-data.sh‎

Lines changed: 118 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -357,71 +357,124 @@ compose exec -T postgres psql -U postgres -d demo_shop -c "SELECT pg_stat_statem
357357
# Run inefficient queries that will be captured by pg_stat_statements
358358
# These patterns are intentionally suboptimal to trigger index recommendations
359359
# NO pg_sleep - all slowness comes from real inefficient query patterns
360-
echo " Starting workload (this takes about 30-60 seconds)..."
361-
# Generate SQL file with individual statements.
362-
# Queries inside PL/pgSQL DO blocks are NOT tracked separately by pg_stat_statements.
363-
# Each SELECT must be a standalone statement to be tracked individually.
364-
# Using generate_series() to repeat each pattern multiple times within a single query,
365-
# which pg_stat_statements will track and report the cumulative execution time.
366-
367-
compose exec -T postgres psql -U postgres -d demo_shop -v ON_ERROR_STOP=1 <<'EOWORK'
368-
-- Pattern 1: LOWER() on status column defeats index (50 calls)
369-
-- Each row of generate_series triggers a separate evaluation via LATERAL join
370-
SELECT 'LOWER(status) pattern' as pattern, COUNT(*)
371-
FROM generate_series(1, 50) g,
372-
LATERAL (SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100) sub;
373-
374-
-- Pattern 2: Full table scan on audit_log 300K rows (20 calls)
375-
SELECT 'audit_log scan pattern' as pattern, COUNT(*)
376-
FROM generate_series(1, 20) g,
377-
LATERAL (SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days') sub;
378-
379-
-- Pattern 3: Sort on audit_log without index (15 calls)
380-
SELECT 'audit_log sort pattern' as pattern, COUNT(*)
381-
FROM generate_series(1, 15) g,
382-
LATERAL (SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000) sub;
383-
384-
-- Pattern 4: ILIKE with leading wildcard (30 calls)
385-
SELECT 'ILIKE pattern' as pattern, COUNT(*)
386-
FROM generate_series(1, 30) g,
387-
LATERAL (SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%') sub;
388-
389-
-- Pattern 5: Large join order_items (100K) to orders (5K) (15 calls)
390-
SELECT 'large join pattern' as pattern, COUNT(*), SUM(total)
391-
FROM generate_series(1, 15) g,
392-
LATERAL (SELECT COUNT(*) as cnt, SUM(oi.subtotal) as total FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered') sub;
393-
394-
-- Pattern 6: Expensive aggregation with GROUP BY (10 calls)
395-
SELECT 'expensive aggregation pattern' as pattern, COUNT(*)
396-
FROM generate_series(1, 10) g,
397-
LATERAL (
398-
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal)
399-
FROM orders o
400-
JOIN order_items oi ON o.id = oi.order_id
401-
JOIN products p ON oi.product_id = p.id
402-
WHERE o.status NOT IN ('cancelled', 'refunded')
403-
GROUP BY 1, p.id, p.name
404-
ORDER BY 4 DESC
405-
LIMIT 100
406-
) sub;
407-
408-
-- Pattern 7: Missing composite index on (status, payment_status) (40 calls)
409-
SELECT 'composite index pattern' as pattern, COUNT(*)
410-
FROM generate_series(1, 40) g,
411-
LATERAL (SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days') sub;
412-
413-
-- Pattern 8: Correlated subquery - inefficient EXISTS (20 calls)
414-
SELECT 'correlated subquery pattern' as pattern, COUNT(*)
415-
FROM generate_series(1, 20) g,
416-
LATERAL (
417-
SELECT COUNT(*) FROM orders o
418-
WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE')
419-
) sub;
420-
421-
SELECT 'Workload patterns completed' as status;
422-
EOWORK
423-
424-
echo " Workload patterns completed."
360+
echo " Starting workload (this takes about 60-90 seconds)..."
361+
362+
# CRITICAL: pg_stat_statements only tracks STANDALONE SQL statements.
363+
# - Queries inside DO $$ PL/pgSQL blocks are NOT tracked
364+
# - LATERAL subqueries are tracked as part of the outer query, not separately
365+
# - Each SELECT must be a separate statement sent to psql
366+
#
367+
# We generate a SQL file with repeated individual statements and pipe to psql.
368+
# Each statement is tracked separately, and identical statements aggregate
369+
# under the same queryid with cumulative calls and total_exec_time.
370+
371+
workload_sql="$(mktemp)"
372+
cat > "$workload_sql" <<'EOSQL'
373+
-- Pattern 1: LOWER() on status column defeats index
374+
-- This is intentionally inefficient - LOWER() prevents index use
375+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
376+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
377+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
378+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
379+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
380+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
381+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
382+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
383+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
384+
SELECT COUNT(*) FROM orders WHERE LOWER(status) = 'delivered' AND total_amount > 100;
385+
386+
-- Pattern 2: Full table scan on audit_log (300K+ rows)
387+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
388+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
389+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
390+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
391+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
392+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
393+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
394+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
395+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
396+
SELECT COUNT(*) FROM audit_log WHERE table_name = 'orders' AND changed_at > NOW() - INTERVAL '90 days';
397+
398+
-- Pattern 3: Sort on audit_log without index on changed_at
399+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
400+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
401+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
402+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
403+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
404+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
405+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
406+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
407+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
408+
SELECT id FROM audit_log WHERE table_name IN ('orders', 'customers', 'products') ORDER BY changed_at DESC LIMIT 1000;
409+
410+
-- Pattern 4: ILIKE with leading wildcard - cannot use index
411+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
412+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
413+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
414+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
415+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
416+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
417+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
418+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
419+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
420+
SELECT COUNT(*) FROM products WHERE name ILIKE '%widget%' OR description ILIKE '%premium%';
421+
422+
-- Pattern 5: Large join order_items (100K) to orders (5K)
423+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
424+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
425+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
426+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
427+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
428+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
429+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
430+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
431+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
432+
SELECT COUNT(*), SUM(oi.subtotal) FROM order_items oi JOIN orders o ON oi.order_id = o.id WHERE o.status = 'delivered';
433+
434+
-- Pattern 6: Expensive aggregation with GROUP BY across large tables
435+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
436+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
437+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
438+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
439+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
440+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
441+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
442+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
443+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
444+
SELECT DATE_TRUNC('month', o.created_at), p.name, COUNT(*), SUM(oi.subtotal) FROM orders o JOIN order_items oi ON o.id = oi.order_id JOIN products p ON oi.product_id = p.id WHERE o.status NOT IN ('cancelled', 'refunded') GROUP BY 1, p.id, p.name ORDER BY 4 DESC LIMIT 100;
445+
446+
-- Pattern 7: Missing composite index on (status, payment_status)
447+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
448+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
449+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
450+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
451+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
452+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
453+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
454+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
455+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
456+
SELECT COUNT(*) FROM orders WHERE status = 'pending' AND payment_status = 'paid' AND created_at > NOW() - INTERVAL '30 days';
457+
458+
-- Pattern 8: Correlated subquery - inefficient EXISTS on audit_log
459+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
460+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
461+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
462+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
463+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
464+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
465+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
466+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
467+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
468+
SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM audit_log a WHERE a.record_id = o.id AND a.table_name = 'orders' AND a.action = 'UPDATE');
469+
EOSQL
470+
471+
# Copy workload SQL into container and run it
472+
compose cp "$workload_sql" postgres:/tmp/workload.sql
473+
compose exec -T postgres psql -U postgres -d demo_shop -q -f /tmp/workload.sql >/dev/null 2>&1
474+
compose exec -T postgres rm -f /tmp/workload.sql
475+
rm -f "$workload_sql"
476+
477+
echo " Workload patterns completed (80 queries across 8 patterns)."
425478

426479
echo " Workload simulation completed."
427480
echo " Verifying pg_stat_statements data..."

0 commit comments

Comments
 (0)