diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ffba985..1c51b1f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,27 @@ true until the next version shipped. 0 on a correct document. The mutation then reddened it as well, which looks like a working removal proof and is two failures agreeing. Re-anchored on a phrase that fits one line; the control is green and the mutation still reddens. +- A covering projection scan was priced at half the base scan for every + restriction, then from every restriction's selectivity. + + `PgColumnarSetRelPathlist` offers a covering-projection path when a + projection stores every referenced column and its leading sort key appears + in a restriction. That path took the base custom-scan run cost and + multiplied by 0.5. The constant does not depend on selectivity, so a 5 + percent range on the sort key was quoted the same as a 50 percent range. + Measured on a 20,000-row table with scrambled insert order: both plans + reported run-cost ratio 0.500 against the base scan. + + Replacing the constant with `rel->rows / rel->tuples` moved with + selectivity, but `rel->rows` is the estimate after every restriction. + The projection prunes only on `sortKey[0]`. A full-range sort key plus a + rare non-sort-key column was then quoted at 0.077 of the base for work + the sort order cannot reduce. The run cost now follows + `clauselist_selectivity` over the clauses that reference that sort key, + floored at one written stripe, and is not discounted twice when the heap + layout already prunes as tightly. The same misattributed query reports + 1.000; a 5 percent range on the sort key still reports 0.050 against a + 50 percent range at 0.500. - A `CONFLICTING` badge on a changelog entry is GitHub, not git (#1116). diff --git a/src/columnar_customscan.c b/src/columnar_customscan.c index 93838c63..80fc813c 100644 --- a/src/columnar_customscan.c +++ b/src/columnar_customscan.c @@ -1621,8 +1621,38 @@ pgcolumnar_sorted_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid relid) * projection name (palloc'd) or NULL. A system-column or whole-row * reference disqualifies a projection scan. */ +static Selectivity +pgcolumnar_sortkey_selectivity(PlannerInfo *root, RelOptInfo *rel, + AttrNumber sortAttno) +{ + List *clauses = NIL; + ListCell *lc; + + /* + * Eligibility tests sortKey[0] against restrictCols. Pricing has to use + * the same clauses: rel->rows is the estimate after EVERY restriction, + * including columns the projection cannot prune on. + */ + if (sortAttno <= 0) + return (Selectivity) 1.0; + + foreach(lc, rel->baserestrictinfo) + { + RestrictInfo *ri = lfirst_node(RestrictInfo, lc); + Bitmapset *cols = NULL; + + pull_varattnos((Node *) ri->clause, rel->relid, &cols); + if (bms_is_member(sortAttno - FirstLowInvalidHeapAttributeNumber, cols)) + clauses = lappend(clauses, ri); + } + if (clauses == NIL) + return (Selectivity) 1.0; + return clauselist_selectivity(root, clauses, rel->relid, JOIN_INNER, NULL); +} + static char * -pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid) +pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid, + AttrNumber *sortAttnoOut) { uint64 storageId; Relation r; @@ -1635,6 +1665,9 @@ pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid) int x; bool haveAdditional = false; + if (sortAttnoOut != NULL) + *sortAttnoOut = 0; + if (!pgcolumnar_enable_projection_scan) return NULL; @@ -1705,6 +1738,8 @@ pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid) { best = pstrdup(p->name); bestNcols = p->columnsLen; + if (sortAttnoOut != NULL) + *sortAttnoOut = p->sortKey[0]; } } @@ -2869,17 +2904,27 @@ PgColumnarSetRelPathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, /* * Offer a projection scan (gap 26) as a competing path when a covering - * projection with a restricted sort key exists. It shares the base scan's - * costs but discounts the run cost, since the sorted per-chunk min/max prunes - * chunks for the sort-key restriction; the planner picks by cost, and the - * result is correct whichever path wins (the executor re-applies the qual). + * projection with a restricted sort key exists. The projection is stored + * sorted on that key, so its run cost follows the sort-key clauses' + * selectivity (one-stripe floor), not a constant 0.5 of the base scan. + * The planner picks by cost; the executor re-applies the qual either way. */ { - char *projName = pgcolumnar_choose_projection(root, rel, rte->relid); + AttrNumber sortAttno = 0; + char *projName = pgcolumnar_choose_projection(root, rel, rte->relid, + &sortAttno); if (projName != NULL) { CustomPath *ppath = makeNode(CustomPath); + Cost serialRun; + Cost projRun; + double sel; + double baseSurvival; + double scale; + double groups; + double floorFrac; + int limit; ppath->path.pathtype = T_CustomScan; ppath->path.parent = rel; @@ -2894,10 +2939,42 @@ PgColumnarSetRelPathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, * freed it. This read is older than #362 and has the same failure * mode -- a projection path costed from freed memory whenever an * index path dominated the base columnar scan. + * + * The base run already includes zonemap survival on the HEAP layout. + * A covering projection is clustered on the restrict key, so replace + * that survival with the selectivity of clauses that reference that + * key -- not rel->rows after every restriction. Dividing by the + * base survival avoids a second discount when the heap is already + * as clustered as the projection. A constant 0.5 made a 5% range + * and a 50% range the same price. */ + serialRun = serialTotalCost - serialStartupCost; + sel = (double) pgcolumnar_sortkey_selectivity(root, rel, sortAttno); + if (sel < 0.0) + sel = 0.0; + if (sel > 1.0) + sel = 1.0; + limit = pgcolumnar_written_stripe_row_limit(rte->relid); + if (limit > 0 && rel->tuples > 0.0) + { + groups = ceil(rel->tuples / (double) limit); + if (groups < 1.0) + groups = 1.0; + floorFrac = 1.0 / groups; + if (sel < floorFrac) + sel = floorFrac; + } + baseSurvival = pgcolumnar_zonemap_survival(rel, rte->relid); + if (baseSurvival < 1e-9) + baseSurvival = 1e-9; + scale = sel / baseSurvival; + if (scale > 1.0) + scale = 1.0; + if (scale < 0.0) + scale = 0.0; + projRun = serialRun * scale; ppath->path.startup_cost = serialStartupCost; - ppath->path.total_cost = serialStartupCost + - (serialTotalCost - serialStartupCost) * 0.5; + ppath->path.total_cost = serialStartupCost + projRun; ppath->path.pathkeys = NIL; ppath->flags = 0; ppath->custom_paths = NIL; diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index d0c69768..e6e0d68a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1374,3 +1374,15 @@ parallel_scan_cost parallel_scan_cost premise: the serial plan has no Gather 15; parallel_scan_cost parallel_scan_cost premise: the serial plan is a columnar scan 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost premise: the table holds every inserted row 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost the leader-participation branch changes the divisor 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost a non-sort-key restriction does not cheapen a covering projection 15;16;17;18;19 2026-09-18 - +projection_scan_cost projection_scan_cost a tight covering projection is cheaper relative to the base than a loose one 15;16;17;18;19 2026-09-17 - +projection_scan_cost projection_scan_cost premise: a covering projection exists 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: every compared scan has a positive run cost 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: every misattributed scan has a positive run cost 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: the loose plan uses the covering projection 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: the misattributed query has a covering projection 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: the table holds every inserted row 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: the tight plan uses the covering projection 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: without the projection scan, the loose plan is a base columnar scan 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost premise: without the projection scan, the tight plan is a base columnar scan 15;16;17;18;19 never - +projection_scan_cost projection_scan_cost tight and loose covering scans are not both priced at half the base 15;16;17;18;19 2026-09-17 - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 2407e4d6..d9f0bac6 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -89,4 +89,14 @@ suites_not_covered 249 # RESEATED onto main carrying #1114 and #1120. Re-derived by the command above on # the merged tree; the ledger auto-merged silently again and was checked by KEY: # 0 main keys lost, 9 added, all in part 070. -checks_never_observed_red 1368 +# 1368 -> 1375 after rebasing projection_scan_cost onto current main: +# nine new ledger rows, of which seven last-red stay never (the two +# load-bearing arms were observed red under the 0.5 causation mutation). +# Re-derived by COUNTING, not by adding seven to a number from another tree: +# awk -F'\t' '$5=="never"' test/check_ledger.tsv | wc -l +# 1375 -> 1377 after the misattributed-selectivity arm: twelve rows, of which +# nine last-red stay never (the three load-bearing arms were observed red: +# two under the 0.5 causation mutation, one under rel->rows / rel->tuples). +# Re-derived by COUNTING, not by adding two to a number from another tree: +# awk -F'\t' '$5=="never"' test/check_ledger.tsv | wc -l +checks_never_observed_red 1377 diff --git a/test/native_join_runtime_filter.sh b/test/native_join_runtime_filter.sh index 5f891909..bcdabf1c 100755 --- a/test/native_join_runtime_filter.sh +++ b/test/native_join_runtime_filter.sh @@ -245,7 +245,7 @@ CREATE TABLE dim_pj(k int); INSERT INTO dim_pj SELECT g FROM generate_series(1,200) g; CREATE TABLE fact_pj(k int, payload text) USING pgcolumnar; SELECT pgcolumnar.set_options($t$fact_pj$t$, stripe_row_limit => 1000); -INSERT INTO fact_pj SELECT g, repeat(md5(g::text), 4) FROM generate_series(1,4000) g; +INSERT INTO fact_pj SELECT g, repeat(md5(g::text), 4) FROM generate_series(1,4000) g ORDER BY md5(g::text); SELECT pgcolumnar.add_projection($t$fact_pj$t$, $n$byk$n$, ARRAY['k','payload'], ARRAY['k']); CREATE TABLE heap_pj AS SELECT * FROM fact_pj; ANALYZE dim_pj; diff --git a/test/projection_scan_cost.sh b/test/projection_scan_cost.sh new file mode 100755 index 00000000..1d9fe75e --- /dev/null +++ b/test/projection_scan_cost.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# pgColumnar: a covering projection scan must not be priced at half the base. +# +# PgColumnarSetRelPathlist offers a covering-projection path by taking the +# base custom-scan run cost and multiplying by 0.5. That constant does not +# depend on the restriction, so a 5% range on the sort key is quoted the +# same as a 50% range. The projection is stored sorted on that key; the +# planner number has to move with selectivity, the way zone-map survival +# already does for the base scan. +# +# This suite pins the PLANNER number, not a runtime. Independent of +# test/pytest/test_projection_scan_cost.py: same public seam (EXPLAIN of a +# columnar scan with and without pgcolumnar.enable_projection_scan), own +# fixture, own observations. +# +# Usage: test/projection_scan_cost.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/lib/postgresql/18/bin/pg_config}" + +N=20000 +TIGHT_HI=1000 +LOOSE_HI=10000 +psql_run "CREATE TABLE prsc (k int, payload text) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('prsc', stripe_row_limit => 1000, chunk_group_row_limit => 500);" +# Physical order is scrambled so the BASE scan cannot prune on k. The +# covering projection is stored sorted on k, which is the only reason it +# should be cheaper than the base for a range on k. +psql_run "INSERT INTO prsc SELECT k, repeat('x', 64) FROM generate_series(1, $N) k ORDER BY md5(k::text);" +psql_run "SELECT pgcolumnar.add_projection('prsc', 'byk', ARRAY['k'], ARRAY['k']);" +psql_run "ANALYZE prsc;" + +explain_scan() { + # $1 = on|off for pgcolumnar.enable_projection_scan + # $2 = SQL + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -Atq \ + -c "SET max_parallel_workers_per_gather = 0;" \ + -c "SET pgcolumnar.enable_ungrouped_vector_agg = off;" \ + -c "SET pgcolumnar.enable_group_vectorization = off;" \ + -c "SET jit = off;" \ + -c "SET pgcolumnar.enable_projection_scan = $1;" \ + -c "EXPLAIN (COSTS ON) $2" \ + | grep -v '^SET$' +} + +scan_cost_pair() { + echo "$1" | grep -F "Custom Scan (PgColumnarScan)" | head -1 \ + | grep -oE "cost=[0-9.]+\.\.[0-9.]+" | head -1 \ + | sed -E "s/cost=([0-9.]+)\\.\\.([0-9.]+)/\\1 \\2/" +} + +run_of() { + local pair start total + pair="$(scan_cost_pair "$1")" + start="${pair%% *}" + total="${pair##* }" + awk -v t="$total" -v s="$start" "BEGIN{ print t-s }" +} + +SQL_TIGHT="SELECT k FROM prsc WHERE k BETWEEN 1 AND $TIGHT_HI" +SQL_LOOSE="SELECT k FROM prsc WHERE k BETWEEN 1 AND $LOOSE_HI" + +tight_proj="$(explain_scan on "$SQL_TIGHT")" +loose_proj="$(explain_scan on "$SQL_LOOSE")" +tight_base="$(explain_scan off "$SQL_TIGHT")" +loose_base="$(explain_scan off "$SQL_LOOSE")" + +t_proj_run="$(run_of "$tight_proj")" +l_proj_run="$(run_of "$loose_proj")" +t_base_run="$(run_of "$tight_base")" +l_base_run="$(run_of "$loose_base")" + +t_ratio="$(awk -v p="$t_proj_run" -v b="$t_base_run" "BEGIN{ if (b<=0) print 0; else printf \"%.3f\", p/b }")" +l_ratio="$(awk -v p="$l_proj_run" -v b="$l_base_run" "BEGIN{ if (b<=0) print 0; else printf \"%.3f\", p/b }")" + +echo "-- tight proj_run=$t_proj_run base_run=$t_base_run ratio=$t_ratio" +echo "-- loose proj_run=$l_proj_run base_run=$l_base_run ratio=$l_ratio" + +check "premise: the table holds every inserted row" \ + "$(q "SELECT count(*) FROM prsc")" "$N" + +check "premise: a covering projection exists" \ + "$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = 'prsc'::regclass AND name = 'byk'")" "1" + +check "premise: the tight plan uses the covering projection" \ + "$(echo "$tight_proj" | grep -c 'Columnar Projection: byk')" "1" + +check "premise: the loose plan uses the covering projection" \ + "$(echo "$loose_proj" | grep -c 'Columnar Projection: byk')" "1" + +check "premise: without the projection scan, the tight plan is a base columnar scan" \ + "$(echo "$tight_base" | grep -c 'Columnar Projection')" "0" + +check "premise: without the projection scan, the loose plan is a base columnar scan" \ + "$(echo "$loose_base" | grep -c 'Columnar Projection')" "0" + +check "premise: every compared scan has a positive run cost" \ + "$(awk -v a="$t_proj_run" -v b="$t_base_run" -v c="$l_proj_run" -v d="$l_base_run" \ + "BEGIN{ print (a>0 && b>0 && c>0 && d>0) ? \"yes\" : \"no\" }")" "yes" + +# The unfixed path multiplies the whole run by 0.5, so both ratios are 0.500. +# A constant other than 0.5 can dodge the "both halved" pin; it cannot make +# a 5% range cheaper relative to the base than a 50% range. +check "a tight covering projection is cheaper relative to the base than a loose one" \ + "$(awk -v t="$t_ratio" -v l="$l_ratio" "BEGIN{ print (t < l) ? \"tighter\" : \"not\" }")" \ + "tighter" + +check "tight and loose covering scans are not both priced at half the base" \ + "$(awk -v t="$t_ratio" -v l="$l_ratio" "BEGIN{ + both = (t>0.45 && t<0.55 && l>0.45 && l<0.55); + print both ? \"both-halved\" : \"scaled\" + }")" "scaled" + + +# The projection prunes only on its sort key. A query whose selectivity comes +# from a different column must not be priced as if the sort order produced +# that selectivity. Own table, own N, own column names; not derived from the +# pytest twin. +N_ATTR=20000 +psql_run "CREATE TABLE prsk (sk int, kind text) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('prsk', stripe_row_limit => 1000, chunk_group_row_limit => 500);" +psql_run "INSERT INTO prsk SELECT sk, CASE WHEN sk % 1000 = 0 THEN 'odd' ELSE 'usual' END FROM generate_series(1, $N_ATTR) sk ORDER BY md5(sk::text);" +psql_run "SELECT pgcolumnar.add_projection('prsk', 'onsk', ARRAY['sk','kind'], ARRAY['sk']);" +psql_run "ANALYZE prsk;" + +SQL_MIS="SELECT sk FROM prsk WHERE sk BETWEEN 1 AND $N_ATTR AND kind = 'odd'" +mis_proj="$(explain_scan on "$SQL_MIS")" +mis_base="$(explain_scan off "$SQL_MIS")" +m_proj_run="$(run_of "$mis_proj")" +m_base_run="$(run_of "$mis_base")" +m_ratio="$(awk -v p="$m_proj_run" -v b="$m_base_run" "BEGIN{ if (b<=0) print 0; else printf \"%.3f\", p/b }")" +echo "-- misattr proj_run=$m_proj_run base_run=$m_base_run ratio=$m_ratio" + +check "premise: the misattributed query has a covering projection" \ + "$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = 'prsk'::regclass AND name = 'onsk'")" "1" + +check "premise: every misattributed scan has a positive run cost" \ + "$(awk -v a="$m_proj_run" -v b="$m_base_run" "BEGIN{ print (a>0 && b>0) ? \"yes\" : \"no\" }")" "yes" + +# rel->rows after every restriction makes this cheap (one-stripe floor over +# heap survival on kind). The sort key is the whole table, so the ratio +# has to sit with the base. +check "a non-sort-key restriction does not cheapen a covering projection" \ + "$(awk -v r="$m_ratio" "BEGIN{ print (r+0 >= 0.8) ? \"not-cheap\" : \"cheap\" }")" \ + "not-cheap" + +pgc_summary diff --git a/test/projection_update.sh b/test/projection_update.sh index 7959d35a..10efe7a1 100755 --- a/test/projection_update.sh +++ b/test/projection_update.sh @@ -17,8 +17,10 @@ set -uo pipefail pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" psql_run "CREATE TABLE pu (a int, b text, c int) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('pu', stripe_row_limit => 2000, chunk_group_row_limit => 1000);" psql_run "SELECT pgcolumnar.add_projection('pu', 'pc', ARRAY['a','c'], ARRAY['c']);" psql_run "INSERT INTO pu SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;" +psql_run "ANALYZE pu;" psql_run "CREATE TABLE pu_h (a int, b text, c int) USING heap;" psql_run "INSERT INTO pu_h SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;" diff --git a/test/projections.sh b/test/projections.sh index 21518cb2..77e8f2f9 100755 --- a/test/projections.sh +++ b/test/projections.sh @@ -177,8 +177,10 @@ check "reconstruct row count matches base" \ # --------------------------------------------------------------------------- echo "-- phase 4b: planner selects a covering projection for a sort-key predicate" psql_run "CREATE TABLE ps (a int, b text, c int) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('ps', stripe_row_limit => 1000, chunk_group_row_limit => 500);" psql_run "SELECT pgcolumnar.add_projection('ps', 'pc', ARRAY['a','c'], ARRAY['c']);" psql_run "INSERT INTO ps SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;" +psql_run "ANALYZE ps;" psql_run "CREATE TABLE ps_h (a int, b text, c int) USING heap;" psql_run "INSERT INTO ps_h SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;" @@ -214,8 +216,10 @@ check "full-range projection scan matches oracle" \ # --------------------------------------------------------------------------- echo "-- phase 5: pgcolumnar.vacuum rebuilds projections aligned to the compacted base" psql_run "CREATE TABLE pv (a int, b text, c int) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('pv', stripe_row_limit => 1000, chunk_group_row_limit => 500);" psql_run "SELECT pgcolumnar.add_projection('pv', 'pvp', ARRAY['a','c'], ARRAY['c']);" psql_run "INSERT INTO pv SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;" +psql_run "ANALYZE pv;" psql_run "DELETE FROM pv WHERE a BETWEEN 5000 AND 8000;" psql_run "CREATE TABLE pv_h (a int, b text, c int) USING heap;" psql_run "INSERT INTO pv_h SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g WHERE g NOT BETWEEN 5000 AND 8000;" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index df83c4be..eafc8f69 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -96,6 +96,7 @@ behaviour, the source of that number is named. - [48. test_parallel_scan_cost.py: a parallel custom scan must not divide I/O](#48-test_parallel_scan_costpy-a-parallel-custom-scan-must-not-divide-io) - [49. test_residual_is_counted.py: a residual must be counted, not subtracted](#49-test_residual_is_countedpy-a-residual-must-be-counted-not-subtracted) - [50. test_collation_pinned.py: comm's inputs must be sorted the same way](#50-test_collation_pinnedpy-comms-inputs-must-be-sorted-the-same-way) +- [51. test_projection_scan_cost.py: a covering projection is not priced at half](#51-test_projection_scan_costpy-a-covering-projection-is-not-priced-at-half) ## 1. How to read a test in here @@ -4576,3 +4577,21 @@ Both corpus arms report zero on this tree, measured before the file was written, the detector is proved by planting rather than by the corpus. The load-bearing arm is the process-substituted form: unreachable by a pipe pattern, and reachable only once the detector reads substitutions too. + +## 51. test_projection_scan_cost.py: a covering projection is not priced at half + +The covering-projection path took the base custom-scan run cost and multiplied +by 0.5. That constant does not depend on the restriction, so a tight range on +the sort key was quoted the same as a loose one. The projection is stored +sorted on that key; the planner number has to move with selectivity. + +This file asserts the PLANNER ratio, not a runtime. Public seam: `EXPLAIN` of +a columnar scan with `pgcolumnar.enable_projection_scan` on and off. The +shell twin uses its own table (`prsc`, 20000 rows, 5 percent vs 50 percent); +this file uses `pscost`, 24000 rows, 1800 vs 12000. The misattributed-selectivity +arm uses `prsk`/`kind` on the shell side and `psmis`/`flag` here. Assertion +names match. + +| test | what it asserts | +| --- | --- | +| `test_projection_scan_cost` | the table and covering projection exist; the tight and loose plans use that projection; without the GUC they are base columnar scans; every compared scan has a positive run cost; a tight covering projection is cheaper relative to the base than a loose one; the two ratios are not both 0.5; a non-sort-key restriction does not cheapen a covering projection | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index b5673c38..cf5fded5 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -279,4 +279,12 @@ guard_tests 374 # the expected answer for a file that needs a cluster, and checking it was the point # rather than assuming it. # cluster_tests re-derived by collection on the rebased tree, never by adding a delta measured on another tree. -cluster_tests 417 +# 417 -> 418 when test_projection_scan_cost.py landed: one cluster arm that +# a covering projection scan is not priced at half the base for every +# restriction. Re-derived by collection on this tree, never by adding one +# to a number measured on another: `418 tests collected`. `guard_tests` +# was re-derived in the same run and did NOT move -- 346. +# After rebase onto origin/main (ab8feef), re-derived by collection on this +# tree, not by keeping the auto-merged 418: guard_tests 374 tests collected; +# cluster_tests 418 tests collected. guard_tests did not move. +cluster_tests 418 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 938f08f0..db54ab30 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -88,6 +88,7 @@ COMPLETE = ["differential", "hilbert_cluster", "hilbert_locality", "native_chunk_length_bound", "native_fetch_coalesce", "native_ownership", "native_projection", "parallel_am_scan", "projection_privilege", + "projection_scan_cost", "projections", "sorted_pathkeys", "stats_privilege", "index_fetch_penalty_crossover", diff --git a/test/pytest/test_join_runtime_filter.py b/test/pytest/test_join_runtime_filter.py index 82b973e3..019ceff1 100644 --- a/test/pytest/test_join_runtime_filter.py +++ b/test/pytest/test_join_runtime_filter.py @@ -552,7 +552,7 @@ def test_projection_outer_is_not_wrapped(pgc_conn, expect): CREATE TABLE factp(k int, payload text) USING pgcolumnar; SELECT pgcolumnar.set_options($t$factp$t$, stripe_row_limit => 1000); INSERT INTO factp SELECT g, repeat(md5(g::text), 4) - FROM generate_series(1, 6000) g; + FROM generate_series(1, 6000) g ORDER BY md5((g + 9)::text); SELECT pgcolumnar.add_projection( $t$factp$t$, $n$pk$n$, ARRAY['k','payload'], ARRAY['k']); CREATE TABLE heapp AS SELECT * FROM factp; diff --git a/test/pytest/test_projection_scan_cost.py b/test/pytest/test_projection_scan_cost.py new file mode 100644 index 00000000..7f7c8b12 --- /dev/null +++ b/test/pytest/test_projection_scan_cost.py @@ -0,0 +1,191 @@ +"""A covering projection scan must not be priced at half the base. + +PgColumnarSetRelPathlist offers a covering-projection path by taking the +base custom-scan run cost and multiplying by 0.5. That constant does not +depend on the restriction, so two ranges on the sort key are quoted at +the same fraction of the base. + +This file asserts the PLANNER ratio, not a runtime. Independent of +test/projection_scan_cost.sh: same public seam (EXPLAIN of a columnar +scan with the projection-scan GUC on and off), own fixture, own +observations. Assertion names match the shell suite so the two can be +compared by name, not by importing each other. +""" + + +def _nodes(plan): + stack = [plan[0]["Plan"]] + while stack: + node = stack.pop(0) + yield node + stack.extend(node.get("Plans") or ()) + + +def _custom_scan(plan): + for node in _nodes(plan): + if node.get("Node Type") == "Custom Scan": + return node + return None + + +def _plan(conn, sql, projection_scan): + with conn.cursor() as cur: + cur.execute("SET max_parallel_workers_per_gather = 0") + cur.execute("SET pgcolumnar.enable_ungrouped_vector_agg = off") + cur.execute("SET pgcolumnar.enable_group_vectorization = off") + cur.execute("SET jit = off") + cur.execute( + "SET pgcolumnar.enable_projection_scan = " + + ("on" if projection_scan else "off") + ) + cur.execute("EXPLAIN (FORMAT JSON, COSTS ON) " + sql) + return cur.fetchone()[0] + + +def test_projection_scan_cost(pgc_conn, expect): + n = 24000 + tight_hi = 1800 + loose_hi = 12000 + with pgc_conn.cursor() as cur: + cur.execute("CREATE TABLE pscost (k int, blob text) USING pgcolumnar") + cur.execute( + "SELECT pgcolumnar.set_options('pscost', stripe_row_limit => 1200, " + "chunk_group_row_limit => 400)" + ) + # Scrambled insert order so the base layout cannot prune on k. + # Different N, stripe, payload, and bounds from the shell twin. + cur.execute( + f"INSERT INTO pscost SELECT k, md5(k::text) FROM generate_series(1, {n}) k " + "ORDER BY md5((k + 17)::text)" + ) + cur.execute( + "SELECT pgcolumnar.add_projection('pscost', 'onk', ARRAY['k'], ARRAY['k'])" + ) + cur.execute("ANALYZE pscost") + cur.execute("SELECT count(*) FROM pscost") + expect.num(cur.fetchone()[0], n, "premise: the table holds every inserted row") + cur.execute( + "SELECT count(*) FROM pgcolumnar.projection_declaration " + "WHERE rel = 'pscost'::regclass AND name = 'onk'" + ) + expect.num(cur.fetchone()[0], 1, "premise: a covering projection exists") + + sql_tight = f"SELECT k FROM pscost WHERE k BETWEEN 1 AND {tight_hi}" + sql_loose = f"SELECT k FROM pscost WHERE k BETWEEN 1 AND {loose_hi}" + + tight_proj = _plan(pgc_conn, sql_tight, True) + loose_proj = _plan(pgc_conn, sql_loose, True) + tight_base = _plan(pgc_conn, sql_tight, False) + loose_base = _plan(pgc_conn, sql_loose, False) + + tp = _custom_scan(tight_proj) + lp = _custom_scan(loose_proj) + tb = _custom_scan(tight_base) + lb = _custom_scan(loose_base) + + expect.text( + (tp or {}).get("Columnar Projection") or "none", + "onk", + "premise: the tight plan uses the covering projection", + ) + expect.text( + (lp or {}).get("Columnar Projection") or "none", + "onk", + "premise: the loose plan uses the covering projection", + ) + expect.text( + "none" if tb is None or "Columnar Projection" not in tb else tb["Columnar Projection"], + "none", + "premise: without the projection scan, the tight plan is a base columnar scan", + ) + expect.text( + "none" if lb is None or "Columnar Projection" not in lb else lb["Columnar Projection"], + "none", + "premise: without the projection scan, the loose plan is a base columnar scan", + ) + + t_proj_run = tp["Total Cost"] - tp["Startup Cost"] + l_proj_run = lp["Total Cost"] - lp["Startup Cost"] + t_base_run = tb["Total Cost"] - tb["Startup Cost"] + l_base_run = lb["Total Cost"] - lb["Startup Cost"] + expect.text( + "yes" if min(t_proj_run, l_proj_run, t_base_run, l_base_run) > 0 else "no", + "yes", + "premise: every compared scan has a positive run cost", + ) + + t_ratio = t_proj_run / t_base_run + l_ratio = l_proj_run / l_base_run + print( + f"-- tight proj_run={t_proj_run} base_run={t_base_run} ratio={t_ratio:.3f}" + ) + print( + f"-- loose proj_run={l_proj_run} base_run={l_base_run} ratio={l_ratio:.3f}" + ) + + expect.text( + "tighter" if t_ratio < l_ratio else "not", + "tighter", + "a tight covering projection is cheaper relative to the base than a loose one", + ) + both_halved = ( + 0.45 < t_ratio < 0.55 and 0.45 < l_ratio < 0.55 + ) + expect.text( + "both-halved" if both_halved else "scaled", + "scaled", + "tight and loose covering scans are not both priced at half the base", + ) + + # Independent of the shell twin: different table, N, stripe, column names, + # and rare-value density. Same public EXPLAIN seam. + n_attr = 30000 + with pgc_conn.cursor() as cur: + cur.execute("CREATE TABLE psmis (ikey int, flag text) USING pgcolumnar") + cur.execute( + "SELECT pgcolumnar.set_options('psmis', stripe_row_limit => 1500, " + "chunk_group_row_limit => 500)" + ) + cur.execute( + f"INSERT INTO psmis SELECT ikey, " + f"CASE WHEN ikey % 1500 = 0 THEN 'x' ELSE 'y' END " + f"FROM generate_series(1, {n_attr}) ikey " + "ORDER BY md5((ikey + 41)::text)" + ) + cur.execute( + "SELECT pgcolumnar.add_projection('psmis', 'onikey', " + "ARRAY['ikey','flag'], ARRAY['ikey'])" + ) + cur.execute("ANALYZE psmis") + cur.execute( + "SELECT count(*) FROM pgcolumnar.projection_declaration " + "WHERE rel = 'psmis'::regclass AND name = 'onikey'" + ) + expect.num( + cur.fetchone()[0], + 1, + "premise: the misattributed query has a covering projection", + ) + + sql_mis = ( + f"SELECT ikey FROM psmis WHERE ikey BETWEEN 1 AND {n_attr} " + "AND flag = 'x'" + ) + mis_proj = _plan(pgc_conn, sql_mis, True) + mis_base = _plan(pgc_conn, sql_mis, False) + mp = _custom_scan(mis_proj) + mb = _custom_scan(mis_base) + m_proj_run = mp["Total Cost"] - mp["Startup Cost"] + m_base_run = mb["Total Cost"] - mb["Startup Cost"] + expect.text( + "yes" if min(m_proj_run, m_base_run) > 0 else "no", + "yes", + "premise: every misattributed scan has a positive run cost", + ) + m_ratio = m_proj_run / m_base_run + print(f"-- misattr proj_run={m_proj_run} base_run={m_base_run} ratio={m_ratio:.3f}") + expect.text( + "not-cheap" if m_ratio >= 0.8 else "cheap", + "not-cheap", + "a non-sort-key restriction does not cheapen a covering projection", + ) diff --git a/test/pytest/test_projections.py b/test/pytest/test_projections.py index 914c53e1..8be8bfda 100644 --- a/test/pytest/test_projections.py +++ b/test/pytest/test_projections.py @@ -435,10 +435,13 @@ def test_reconstruction_survives_deletes_and_nulls(recon, expect): def planner(pgc_conn): with pgc_conn.cursor() as cur: cur.execute("CREATE TABLE ps (a int, b text, c int) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('ps', stripe_row_limit => 2500, " + "chunk_group_row_limit => 1000)") cur.execute("SELECT pgcolumnar.add_projection('ps','pc'," "ARRAY['a','c'],ARRAY['c'])") cur.execute("INSERT INTO ps SELECT g, 'r'||g, (g*7)%%1000 " "FROM generate_series(1,%s) g", (PLANNER_ROWS,)) + cur.execute("ANALYZE ps") cur.execute("CREATE TABLE ps_h (a int, b text, c int) USING heap") cur.execute("INSERT INTO ps_h SELECT g, 'r'||g, (g*7)%%1000 " "FROM generate_series(1,%s) g", (PLANNER_ROWS,)) @@ -507,11 +510,14 @@ def test_a_projection_scan_reflects_deletes(planner, expect): def vac(pgc_conn): with pgc_conn.cursor() as cur: cur.execute("CREATE TABLE pv (a int, b text, c int) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('pv', stripe_row_limit => 2500, " + "chunk_group_row_limit => 1000)") cur.execute("SELECT pgcolumnar.add_projection('pv','pvp'," "ARRAY['a','c'],ARRAY['c'])") cur.execute("INSERT INTO pv SELECT g, 'r'||g, (g*7)%%1000 " "FROM generate_series(1,%s) g", (PLANNER_ROWS,)) cur.execute("DELETE FROM pv WHERE a BETWEEN 5000 AND 8000") + cur.execute("ANALYZE pv") cur.execute("CREATE TABLE pv_h (a int, b text, c int) USING heap") cur.execute("INSERT INTO pv_h SELECT g, 'r'||g, (g*7)%%1000 " "FROM generate_series(1,%s) g WHERE g NOT BETWEEN 5000 AND 8000", diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index d3e2724f..da247986 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -254,6 +254,7 @@ SUITES=( projection_privilege projection_rename_restore projection_rewrite + projection_scan_cost projection_update projections pushdown_report