fix: price a covering projection scan from selectivity, not half

CHANGELOG.md

@@ -41,6 +41,27 @@ true until the next version shipped.
   0 on a correct document. The mutation then reddened it as well, which looks like a
   working removal proof and is two failures agreeing. Re-anchored on a phrase that
   fits one line; the control is green and the mutation still reddens.
+- A covering projection scan was priced at half the base scan for every
+  restriction, then from every restriction's selectivity.
+
+  `PgColumnarSetRelPathlist` offers a covering-projection path when a
+  projection stores every referenced column and its leading sort key appears
+  in a restriction. That path took the base custom-scan run cost and
+  multiplied by 0.5. The constant does not depend on selectivity, so a 5
+  percent range on the sort key was quoted the same as a 50 percent range.
+  Measured on a 20,000-row table with scrambled insert order: both plans
+  reported run-cost ratio 0.500 against the base scan.
+
+  Replacing the constant with `rel->rows / rel->tuples` moved with
+  selectivity, but `rel->rows` is the estimate after every restriction.
+  The projection prunes only on `sortKey[0]`. A full-range sort key plus a
+  rare non-sort-key column was then quoted at 0.077 of the base for work
+  the sort order cannot reduce. The run cost now follows
+  `clauselist_selectivity` over the clauses that reference that sort key,
+  floored at one written stripe, and is not discounted twice when the heap
+  layout already prunes as tightly. The same misattributed query reports
+  1.000; a 5 percent range on the sort key still reports 0.050 against a
+  50 percent range at 0.500.
 
 - A `CONFLICTING` badge on a changelog entry is GitHub, not git (#1116).
 

src/columnar_customscan.c

@@ -1621,8 +1621,38 @@ pgcolumnar_sorted_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid relid)
  *		projection name (palloc'd) or NULL. A system-column or whole-row
  *		reference disqualifies a projection scan.
  */
+static Selectivity
+pgcolumnar_sortkey_selectivity(PlannerInfo *root, RelOptInfo *rel,
+							   AttrNumber sortAttno)
+{
+	List	   *clauses = NIL;
+	ListCell   *lc;
+
+	/*
+	 * Eligibility tests sortKey[0] against restrictCols. Pricing has to use
+	 * the same clauses: rel->rows is the estimate after EVERY restriction,
+	 * including columns the projection cannot prune on.
+	 */
+	if (sortAttno <= 0)
+		return (Selectivity) 1.0;
+
+	foreach(lc, rel->baserestrictinfo)
+	{
+		RestrictInfo *ri = lfirst_node(RestrictInfo, lc);
+		Bitmapset  *cols = NULL;
+
+		pull_varattnos((Node *) ri->clause, rel->relid, &cols);
+		if (bms_is_member(sortAttno - FirstLowInvalidHeapAttributeNumber, cols))
+			clauses = lappend(clauses, ri);
+	}
+	if (clauses == NIL)
+		return (Selectivity) 1.0;
+	return clauselist_selectivity(root, clauses, rel->relid, JOIN_INNER, NULL);
+}
+
 static char *
-pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid)
+pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid,
+							 AttrNumber *sortAttnoOut)
 {
 	uint64		storageId;
 	Relation	r;
@@ -1635,6 +1665,9 @@ pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid)
 	int			x;
 	bool		haveAdditional = false;
 
+	if (sortAttnoOut != NULL)
+		*sortAttnoOut = 0;
+
 	if (!pgcolumnar_enable_projection_scan)
 		return NULL;
 
@@ -1705,6 +1738,8 @@ pgcolumnar_choose_projection(PlannerInfo *root, RelOptInfo *rel, Oid relid)
 		{
 			best = pstrdup(p->name);
 			bestNcols = p->columnsLen;
+			if (sortAttnoOut != NULL)
+				*sortAttnoOut = p->sortKey[0];
 		}
 	}
 
@@ -2869,17 +2904,27 @@ PgColumnarSetRelPathlist(PlannerInfo *root, RelOptInfo *rel, Index rti,
 
 	/*
 	 * Offer a projection scan (gap 26) as a competing path when a covering
-	 * projection with a restricted sort key exists. It shares the base scan's
-	 * costs but discounts the run cost, since the sorted per-chunk min/max prunes
-	 * chunks for the sort-key restriction; the planner picks by cost, and the
-	 * result is correct whichever path wins (the executor re-applies the qual).
+	 * projection with a restricted sort key exists. The projection is stored
+	 * sorted on that key, so its run cost follows the sort-key clauses'
+	 * selectivity (one-stripe floor), not a constant 0.5 of the base scan.
+	 * The planner picks by cost; the executor re-applies the qual either way.
 	 */
 	{
-		char	   *projName = pgcolumnar_choose_projection(root, rel, rte->relid);
+		AttrNumber	sortAttno = 0;
+		char	   *projName = pgcolumnar_choose_projection(root, rel, rte->relid,
+															&sortAttno);
 
 		if (projName != NULL)
 		{
 			CustomPath *ppath = makeNode(CustomPath);
+			Cost		serialRun;
+			Cost		projRun;
+			double		sel;
+			double		baseSurvival;
+			double		scale;
+			double		groups;
+			double		floorFrac;
+			int			limit;
 
 			ppath->path.pathtype = T_CustomScan;
 			ppath->path.parent = rel;
@@ -2894,10 +2939,42 @@ PgColumnarSetRelPathlist(PlannerInfo *root, RelOptInfo *rel, Index rti,
 			 * freed it. This read is older than #362 and has the same failure
 			 * mode -- a projection path costed from freed memory whenever an
 			 * index path dominated the base columnar scan.
+			 *
+			 * The base run already includes zonemap survival on the HEAP layout.
+			 * A covering projection is clustered on the restrict key, so replace
+			 * that survival with the selectivity of clauses that reference that
+			 * key -- not rel->rows after every restriction. Dividing by the
+			 * base survival avoids a second discount when the heap is already
+			 * as clustered as the projection. A constant 0.5 made a 5% range
+			 * and a 50% range the same price.
 			 */
+			serialRun = serialTotalCost - serialStartupCost;
+			sel = (double) pgcolumnar_sortkey_selectivity(root, rel, sortAttno);
+			if (sel < 0.0)
+				sel = 0.0;
+			if (sel > 1.0)
+				sel = 1.0;
+			limit = pgcolumnar_written_stripe_row_limit(rte->relid);
+			if (limit > 0 && rel->tuples > 0.0)
+			{
+				groups = ceil(rel->tuples / (double) limit);
+				if (groups < 1.0)
+					groups = 1.0;
+				floorFrac = 1.0 / groups;
+				if (sel < floorFrac)
+					sel = floorFrac;
+			}
+			baseSurvival = pgcolumnar_zonemap_survival(rel, rte->relid);
+			if (baseSurvival < 1e-9)
+				baseSurvival = 1e-9;
+			scale = sel / baseSurvival;
+			if (scale > 1.0)
+				scale = 1.0;
+			if (scale < 0.0)
+				scale = 0.0;
+			projRun = serialRun * scale;
 			ppath->path.startup_cost = serialStartupCost;
-			ppath->path.total_cost = serialStartupCost +
-				(serialTotalCost - serialStartupCost) * 0.5;
+			ppath->path.total_cost = serialStartupCost + projRun;
 			ppath->path.pathkeys = NIL;
 			ppath->flags = 0;
 			ppath->custom_paths = NIL;

test/check_ledger.tsv

@@ -1374,3 +1374,15 @@ parallel_scan_cost	parallel_scan_cost	premise: the serial plan has no Gather	15;
 parallel_scan_cost	parallel_scan_cost	premise: the serial plan is a columnar scan	15;16;17;18;19	never	-
 parallel_scan_cost	parallel_scan_cost	premise: the table holds every inserted row	15;16;17;18;19	never	-
 parallel_scan_cost	parallel_scan_cost	the leader-participation branch changes the divisor	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	a non-sort-key restriction does not cheapen a covering projection	15;16;17;18;19	2026-09-18	-
+projection_scan_cost	projection_scan_cost	a tight covering projection is cheaper relative to the base than a loose one	15;16;17;18;19	2026-09-17	-
+projection_scan_cost	projection_scan_cost	premise: a covering projection exists	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: every compared scan has a positive run cost	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: every misattributed scan has a positive run cost	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: the loose plan uses the covering projection	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: the misattributed query has a covering projection	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: the table holds every inserted row	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: the tight plan uses the covering projection	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: without the projection scan, the loose plan is a base columnar scan	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	premise: without the projection scan, the tight plan is a base columnar scan	15;16;17;18;19	never	-
+projection_scan_cost	projection_scan_cost	tight and loose covering scans are not both priced at half the base	15;16;17;18;19	2026-09-17	-

test/check_ledger_budget.txt

@@ -89,4 +89,14 @@ suites_not_covered 249
 # RESEATED onto main carrying #1114 and #1120. Re-derived by the command above on
 # the merged tree; the ledger auto-merged silently again and was checked by KEY:
 # 0 main keys lost, 9 added, all in part 070.
-checks_never_observed_red 1368
+# 1368 -> 1375 after rebasing projection_scan_cost onto current main:
+# nine new ledger rows, of which seven last-red stay never (the two
+# load-bearing arms were observed red under the 0.5 causation mutation).
+# Re-derived by COUNTING, not by adding seven to a number from another tree:
+#     awk -F'\t' '$5=="never"' test/check_ledger.tsv | wc -l
+# 1375 -> 1377 after the misattributed-selectivity arm: twelve rows, of which
+# nine last-red stay never (the three load-bearing arms were observed red:
+# two under the 0.5 causation mutation, one under rel->rows / rel->tuples).
+# Re-derived by COUNTING, not by adding two to a number from another tree:
+#     awk -F'\t' '$5=="never"' test/check_ledger.tsv | wc -l
+checks_never_observed_red 1377

test/native_join_runtime_filter.sh

@@ -245,7 +245,7 @@ CREATE TABLE dim_pj(k int);
 INSERT INTO dim_pj SELECT g FROM generate_series(1,200) g;
 CREATE TABLE fact_pj(k int, payload text) USING pgcolumnar;
 SELECT pgcolumnar.set_options($t$fact_pj$t$, stripe_row_limit => 1000);
-INSERT INTO fact_pj SELECT g, repeat(md5(g::text), 4) FROM generate_series(1,4000) g;
+INSERT INTO fact_pj SELECT g, repeat(md5(g::text), 4) FROM generate_series(1,4000) g ORDER BY md5(g::text);
 SELECT pgcolumnar.add_projection($t$fact_pj$t$, $n$byk$n$, ARRAY['k','payload'], ARRAY['k']);
 CREATE TABLE heap_pj AS SELECT * FROM fact_pj;
 ANALYZE dim_pj;

test/projection_scan_cost.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+#
+# pgColumnar: a covering projection scan must not be priced at half the base.
+#
+# PgColumnarSetRelPathlist offers a covering-projection path by taking the
+# base custom-scan run cost and multiplying by 0.5. That constant does not
+# depend on the restriction, so a 5% range on the sort key is quoted the
+# same as a 50% range. The projection is stored sorted on that key; the
+# planner number has to move with selectivity, the way zone-map survival
+# already does for the base scan.
+#
+# This suite pins the PLANNER number, not a runtime. Independent of
+# test/pytest/test_projection_scan_cost.py: same public seam (EXPLAIN of a
+# columnar scan with and without pgcolumnar.enable_projection_scan), own
+# fixture, own observations.
+#
+# Usage:  test/projection_scan_cost.sh [PG_CONFIG]
+# Written fresh for pgColumnar.
+
+set -uo pipefail
+. "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+pgc_setup "${1:-/usr/lib/postgresql/18/bin/pg_config}"
+
+N=20000
+TIGHT_HI=1000
+LOOSE_HI=10000
+psql_run "CREATE TABLE prsc (k int, payload text) USING pgcolumnar;"
+psql_run "SELECT pgcolumnar.set_options('prsc', stripe_row_limit => 1000, chunk_group_row_limit => 500);"
+# Physical order is scrambled so the BASE scan cannot prune on k. The
+# covering projection is stored sorted on k, which is the only reason it
+# should be cheaper than the base for a range on k.
+psql_run "INSERT INTO prsc SELECT k, repeat('x', 64) FROM generate_series(1, $N) k ORDER BY md5(k::text);"
+psql_run "SELECT pgcolumnar.add_projection('prsc', 'byk', ARRAY['k'], ARRAY['k']);"
+psql_run "ANALYZE prsc;"
+
+explain_scan() {
+	# $1 = on|off for pgcolumnar.enable_projection_scan
+	# $2 = SQL
+	env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \
+		-d "$PGC_DB" -Atq \
+		-c "SET max_parallel_workers_per_gather = 0;" \
+		-c "SET pgcolumnar.enable_ungrouped_vector_agg = off;" \
+		-c "SET pgcolumnar.enable_group_vectorization = off;" \
+		-c "SET jit = off;" \
+		-c "SET pgcolumnar.enable_projection_scan = $1;" \
+		-c "EXPLAIN (COSTS ON) $2" \
+		| grep -v '^SET$'
+}
+
+scan_cost_pair() {
+	echo "$1" | grep -F "Custom Scan (PgColumnarScan)" | head -1 \
+		| grep -oE "cost=[0-9.]+\.\.[0-9.]+" | head -1 \
+		| sed -E "s/cost=([0-9.]+)\\.\\.([0-9.]+)/\\1 \\2/"
+}
+
+run_of() {
+	local pair start total
+	pair="$(scan_cost_pair "$1")"
+	start="${pair%% *}"
+	total="${pair##* }"
+	awk -v t="$total" -v s="$start" "BEGIN{ print t-s }"
+}
+
+SQL_TIGHT="SELECT k FROM prsc WHERE k BETWEEN 1 AND $TIGHT_HI"
+SQL_LOOSE="SELECT k FROM prsc WHERE k BETWEEN 1 AND $LOOSE_HI"
+
+tight_proj="$(explain_scan on "$SQL_TIGHT")"
+loose_proj="$(explain_scan on "$SQL_LOOSE")"
+tight_base="$(explain_scan off "$SQL_TIGHT")"
+loose_base="$(explain_scan off "$SQL_LOOSE")"
+
+t_proj_run="$(run_of "$tight_proj")"
+l_proj_run="$(run_of "$loose_proj")"
+t_base_run="$(run_of "$tight_base")"
+l_base_run="$(run_of "$loose_base")"
+
+t_ratio="$(awk -v p="$t_proj_run" -v b="$t_base_run" "BEGIN{ if (b<=0) print 0; else printf \"%.3f\", p/b }")"
+l_ratio="$(awk -v p="$l_proj_run" -v b="$l_base_run" "BEGIN{ if (b<=0) print 0; else printf \"%.3f\", p/b }")"
+
+echo "-- tight proj_run=$t_proj_run base_run=$t_base_run ratio=$t_ratio"
+echo "-- loose proj_run=$l_proj_run base_run=$l_base_run ratio=$l_ratio"
+
+check "premise: the table holds every inserted row" \
+	"$(q "SELECT count(*) FROM prsc")" "$N"
+
+check "premise: a covering projection exists" \
+	"$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = 'prsc'::regclass AND name = 'byk'")" "1"
+
+check "premise: the tight plan uses the covering projection" \
+	"$(echo "$tight_proj" | grep -c 'Columnar Projection: byk')" "1"
+
+check "premise: the loose plan uses the covering projection" \
+	"$(echo "$loose_proj" | grep -c 'Columnar Projection: byk')" "1"
+
+check "premise: without the projection scan, the tight plan is a base columnar scan" \
+	"$(echo "$tight_base" | grep -c 'Columnar Projection')" "0"
+
+check "premise: without the projection scan, the loose plan is a base columnar scan" \
+	"$(echo "$loose_base" | grep -c 'Columnar Projection')" "0"
+
+check "premise: every compared scan has a positive run cost" \
+	"$(awk -v a="$t_proj_run" -v b="$t_base_run" -v c="$l_proj_run" -v d="$l_base_run" \
+		"BEGIN{ print (a>0 && b>0 && c>0 && d>0) ? \"yes\" : \"no\" }")" "yes"
+
+# The unfixed path multiplies the whole run by 0.5, so both ratios are 0.500.
+# A constant other than 0.5 can dodge the "both halved" pin; it cannot make
+# a 5% range cheaper relative to the base than a 50% range.
+check "a tight covering projection is cheaper relative to the base than a loose one" \
+	"$(awk -v t="$t_ratio" -v l="$l_ratio" "BEGIN{ print (t < l) ? \"tighter\" : \"not\" }")" \
+	"tighter"
+
+check "tight and loose covering scans are not both priced at half the base" \
+	"$(awk -v t="$t_ratio" -v l="$l_ratio" "BEGIN{
+		both = (t>0.45 && t<0.55 && l>0.45 && l<0.55);
+		print both ? \"both-halved\" : \"scaled\"
+	}")" "scaled"
+
+
+# The projection prunes only on its sort key. A query whose selectivity comes
+# from a different column must not be priced as if the sort order produced
+# that selectivity. Own table, own N, own column names; not derived from the
+# pytest twin.
+N_ATTR=20000
+psql_run "CREATE TABLE prsk (sk int, kind text) USING pgcolumnar;"
+psql_run "SELECT pgcolumnar.set_options('prsk', stripe_row_limit => 1000, chunk_group_row_limit => 500);"
+psql_run "INSERT INTO prsk SELECT sk, CASE WHEN sk % 1000 = 0 THEN 'odd' ELSE 'usual' END FROM generate_series(1, $N_ATTR) sk ORDER BY md5(sk::text);"
+psql_run "SELECT pgcolumnar.add_projection('prsk', 'onsk', ARRAY['sk','kind'], ARRAY['sk']);"
+psql_run "ANALYZE prsk;"
+
+SQL_MIS="SELECT sk FROM prsk WHERE sk BETWEEN 1 AND $N_ATTR AND kind = 'odd'"
+mis_proj="$(explain_scan on "$SQL_MIS")"
+mis_base="$(explain_scan off "$SQL_MIS")"
+m_proj_run="$(run_of "$mis_proj")"
+m_base_run="$(run_of "$mis_base")"
+m_ratio="$(awk -v p="$m_proj_run" -v b="$m_base_run" "BEGIN{ if (b<=0) print 0; else printf \"%.3f\", p/b }")"
+echo "-- misattr proj_run=$m_proj_run base_run=$m_base_run ratio=$m_ratio"
+
+check "premise: the misattributed query has a covering projection" \
+	"$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = 'prsk'::regclass AND name = 'onsk'")" "1"
+
+check "premise: every misattributed scan has a positive run cost" \
+	"$(awk -v a="$m_proj_run" -v b="$m_base_run" "BEGIN{ print (a>0 && b>0) ? \"yes\" : \"no\" }")" "yes"
+
+# rel->rows after every restriction makes this cheap (one-stripe floor over
+# heap survival on kind). The sort key is the whole table, so the ratio
+# has to sit with the base.
+check "a non-sort-key restriction does not cheapen a covering projection" \
+	"$(awk -v r="$m_ratio" "BEGIN{ print (r+0 >= 0.8) ? \"not-cheap\" : \"cheap\" }")" \
+	"not-cheap"
+
+pgc_summary

test/projection_update.sh

@@ -17,8 +17,10 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 psql_run "CREATE TABLE pu (a int, b text, c int) USING pgcolumnar;"
+psql_run "SELECT pgcolumnar.set_options('pu', stripe_row_limit => 2000, chunk_group_row_limit => 1000);"
 psql_run "SELECT pgcolumnar.add_projection('pu', 'pc', ARRAY['a','c'], ARRAY['c']);"
 psql_run "INSERT INTO pu SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;"
+psql_run "ANALYZE pu;"
 psql_run "CREATE TABLE pu_h (a int, b text, c int) USING heap;"
 psql_run "INSERT INTO pu_h SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;"
 

test/projections.sh

@@ -177,8 +177,10 @@ check "reconstruct row count matches base" \
 # ---------------------------------------------------------------------------
 echo "-- phase 4b: planner selects a covering projection for a sort-key predicate"
 psql_run "CREATE TABLE ps (a int, b text, c int) USING pgcolumnar;"
+psql_run "SELECT pgcolumnar.set_options('ps', stripe_row_limit => 1000, chunk_group_row_limit => 500);"
 psql_run "SELECT pgcolumnar.add_projection('ps', 'pc', ARRAY['a','c'], ARRAY['c']);"
 psql_run "INSERT INTO ps SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;"
+psql_run "ANALYZE ps;"
 psql_run "CREATE TABLE ps_h (a int, b text, c int) USING heap;"
 psql_run "INSERT INTO ps_h SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;"
 
@@ -214,8 +216,10 @@ check "full-range projection scan matches oracle" \
 # ---------------------------------------------------------------------------
 echo "-- phase 5: pgcolumnar.vacuum rebuilds projections aligned to the compacted base"
 psql_run "CREATE TABLE pv (a int, b text, c int) USING pgcolumnar;"
+psql_run "SELECT pgcolumnar.set_options('pv', stripe_row_limit => 1000, chunk_group_row_limit => 500);"
 psql_run "SELECT pgcolumnar.add_projection('pv', 'pvp', ARRAY['a','c'], ARRAY['c']);"
 psql_run "INSERT INTO pv SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;"
+psql_run "ANALYZE pv;"
 psql_run "DELETE FROM pv WHERE a BETWEEN 5000 AND 8000;"
 psql_run "CREATE TABLE pv_h (a int, b text, c int) USING heap;"
 psql_run "INSERT INTO pv_h SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g WHERE g NOT BETWEEN 5000 AND 8000;"