diff --git a/CHANGELOG.md b/CHANGELOG.md index ef022ea2..7fbbe750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1115,6 +1115,24 @@ Measured by the identity of the postmaster each after them has the same hole. The port asserts the accepted rows are present first. `cluster_tests` 435 -> 436, re-derived by collection. +- A covering projection scan was priced from the base table's pages, so a + column subset stored as its own row groups was quoted as a full-file read. + + `pgcolumnar_scan_io_run_cost` and the covering path in + `PgColumnarSetRelPathlist` used `seq_page_cost * rel->pages`. The relation + file holds the base plus every projection. The covering path now charges + I/O from that projection's own page-rounded row groups, times the sort-key + selectivity (already one-stripe floored). CPU still scales from the base + run: decode work follows the rows, not the file. A lookup that cannot + find that projection's storage falls back to rel->pages rather than + one page, so a failed lookup cannot make the covering path look + cheaper than the base scan. + + Measured on PG18 with `seq_page_cost = 1000` and CPU terms zeroed, 24000 + rows, a covering projection of `(ik, bulky)` beside the base: covering run + 41991.6 against base-page I/O 42000 (ratio 1.000) before, 19996 against + 42000 (ratio 0.476) after. The projection occupied 56366 of 344064 relation + bytes. - A named skip could not be ported: `cannot_run` recorded under its reason code, so 66 suites and 1,281 names could not reach `missing: 0` (#1131, #1150). diff --git a/src/columnar_customscan.c b/src/columnar_customscan.c index 1b5665ef..38f18b52 100644 --- a/src/columnar_customscan.c +++ b/src/columnar_customscan.c @@ -2719,6 +2719,72 @@ pgcolumnar_parallel_divisor(Path *path) return parallel_divisor; } +/* + * pgcolumnar_projection_pages + * Physical pages occupied by one named covering projection's row groups. + * rel->pages is the whole relation file (base plus every projection); + * a covering scan reads only this subset. + * + * The row-group walk is the I/O estimate itself, not a refinement of an + * approximate term, so it earns the plan-time scan that + * pgcolumnar_written_stripe_row_limit declines: without it the covering path + * inherits the whole-file page count and cannot compete honestly. + * + * fallbackPages is rel->pages. A lookup failure must not make the path look + * cheaper than the base scan; returning 1 would. + */ +static BlockNumber +pgcolumnar_projection_pages(Oid relid, const char *projName, + BlockNumber fallbackPages) +{ + Relation r; + uint64 storageId; + uint64 projSid = 0; + List *projs; + ListCell *lc; + List *rgs; + uint64 bytes = 0; + BlockNumber pages; + Snapshot snap; + + r = table_open(relid, AccessShareLock); + storageId = PgColumnarStorageId(r); + table_close(r, AccessShareLock); + + projs = PgColumnarListProjections(storageId); + foreach(lc, projs) + { + PgColumnarProjection *pr = (PgColumnarProjection *) lfirst(lc); + + if (pr->projectionId > 0 && strcmp(pr->name, projName) == 0) + { + projSid = pr->projStorageId; + break; + } + } + if (projSid == 0) + return fallbackPages; /* lookup failed: do not undercut the base file */ + + snap = GetActiveSnapshot(); + if (snap == NULL) + snap = GetTransactionSnapshot(); + rgs = PgColumnarReadRowGroupList(projSid, PgColumnarCatalogSnapshot(snap)); + foreach(lc, rgs) + { + NativeRowGroupMetadata *rg = (NativeRowGroupMetadata *) lfirst(lc); + + bytes += COLUMNAR_PAGE_ROUND_UP(rg->byteLength); + } + pages = (BlockNumber) (bytes / COLUMNAR_BYTES_PER_PAGE); + /* + * A real projection occupying less than a page is genuinely near free. + * This 1 is arithmetic, not the lookup-failure path above. + */ + if (pages < 1) + pages = 1; + return pages; +} + /* * pgcolumnar_scan_io_run_cost * The page-read portion of a columnar scan, after projected-width @@ -3090,7 +3156,20 @@ PgColumnarSetRelPathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, projScale = 1.0; if (projScale < 0.0) projScale = 0.0; - projRun = serialRun * projScale; + { + BlockNumber projPages; + Cost ioBase; + Cost ioProj; + Cost cpuRun; + + projPages = pgcolumnar_projection_pages(rte->relid, projName, rel->pages); + ioBase = pgcolumnar_scan_io_run_cost(rel, rte->relid); + cpuRun = serialRun - ioBase; + if (cpuRun < 0.0) + cpuRun = 0.0; + ioProj = seq_page_cost * (double) projPages * sel; + projRun = cpuRun * projScale + ioProj; + } ppath->path.startup_cost = serialStartupCost; ppath->path.total_cost = serialStartupCost + projRun; ppath->path.pathkeys = NIL; @@ -3282,15 +3361,33 @@ PgColumnarSetRelPathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, prpath->path.parallel_safe = true; prpath->path.parallel_workers = workers; /* - * Clamp ioRunProj to projRun. With projRun = serialRun * - * projScale this is unreachable: ioRun was already clamped - * to serialRun one level up, and multiplying both sides by - * the same non-negative projScale preserves the order. It - * becomes live if projRun is ever computed independently - * (for example from the projection's own pages). When the - * clamp binds fully, cpuRunProj is zero and the partial - * covering path totals exactly like the serial covering - * path, so Gather loses. + * Clamp ioRunProj to projRun. #1127 wrote that this was + * unreachable "with projRun = serialRun * projScale", and + * named "computed independently (for example from the + * projection's own pages)" as what would make it live. + * THIS CHANGE IS THAT, so the premise no longer holds and + * the sentence is corrected rather than carried. + * + * IT IS NOT KNOWN TO BE REACHABLE EITHER, and that is a + * measurement rather than an argument. @OffgridwithJD probed + * it: reached three times in projection_parallel.sh and bound + * zero, with margins 24.4794 against 2122.2110 and 0.2473 + * against 163.8619; a fixture built to bind it reached once + * and still did not, 0.2504 against 148.2537. Binding needs + * + * 2*ioBase - serialRun > baseSurvival * seq_page_cost * projPages + * + * in which sel cancels, and both attempts moved the margin the + * wrong way, by 87x and then 592x, because + * pgcolumnar_scan_io_run_cost prices only the columns read. + * + * So: the old premise is FALSE, and reachability is UNPROVEN. + * The clamp stays because it is cheap and its absence would be + * a silently negative cpuRunProj. + * + * When the clamp binds fully, cpuRunProj is zero and the + * partial covering path totals exactly like the serial + * covering path, so Gather loses. */ ioRunProj = ioRun * projScale; if (ioRunProj > projRun) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 3b645460..c72cb09b 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1499,6 +1499,15 @@ range_pruning range_pruning premise: the range type collates differently from it range_pruning range_pruning so does a containment probe 15;16;17;18;19 never - range_pruning range_pruning the overlap qual is pushed down, where it used to be dropped 15;16;17;18;19 never - range_pruning range_pruning the same for a containment probe far to its right 15;16;17;18;19 never - +projection_scan_io projection_scan_io a covering projection is not priced from the base table's pages 15;16;17;18;19 2026-09-19 rel->pages +projection_scan_io projection_scan_io a covering projection whose storage cannot be found is not priced as one page 15;16;17;18 2026-09-21 return 1 +projection_scan_io projection_scan_io premise: a covering path is still offered when projection storage cannot be found 15;16;17;18 never - +projection_scan_io projection_scan_io premise: a covering projection exists 15;16;17;18;19 never - +projection_scan_io projection_scan_io premise: the covering projection occupies a minority of the relation 15;16;17;18;19 never - +projection_scan_io projection_scan_io premise: the covering scan has a positive run cost 15;16;17;18;19 never - +projection_scan_io projection_scan_io premise: the plan uses the covering projection 15;16;17;18;19 never - +projection_scan_io projection_scan_io premise: the projection row still exists after its storage id is cleared 15;16;17;18 never - +projection_scan_io projection_scan_io premise: the table holds every inserted row 15;16;17;18;19 never - validity_elision validity_elision a chunk claiming no bitmap while it holds fewer values than rows is refused 15;16;17;18;19 never - validity_elision validity_elision a column with no nulls stores no validity bitmap 15;16;17;18;19 2026-09-18 writer: flush_one_column's presentCount == rowCount forced false, so the bitmap is written even when the chunk holds no null validity_elision validity_elision a null-free column elides its bitmap beside a null-bearing one in the same row group 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 844fbe07..ad89a958 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -209,4 +209,6 @@ suites_not_covered 249 # RE-DERIVED ON THE COMPOSED TREE after a rebase (#1127), by counting rather # than by adding this branch's nine rows to either side: main moved while this # sat, so neither the branch's previous value nor main's is the composed one. -checks_never_observed_red 1499 +# 1499 -> 1506 for #1155's own nine rows, re-counted after this branch was +# rebuilt on main rather than added to either side. +checks_never_observed_red 1506 diff --git a/test/projection_scan_io.sh b/test/projection_scan_io.sh new file mode 100755 index 00000000..2678ba14 --- /dev/null +++ b/test/projection_scan_io.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# +# pgColumnar: a covering projection must be priced from its own pages. +# +# PgColumnarSetRelPathlist offers a covering-projection path by scaling the +# BASE scan's run cost. That run's I/O term is seq_page_cost * rel->pages, +# the whole relation file (base plus every projection). A covering projection +# is stored as its own row groups; charging the base page count prices that +# scan as a full-table read of a file that also holds the base copy. +# +# This suite pins the PLANNER number, not a runtime. Independent of +# test/pytest/test_projection_scan_io.py: same public seam (EXPLAIN cost of a +# covering projection vs the relation's pages), own fixture, own observations. +# +# Usage: test/projection_scan_io.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/lib/postgresql/18/bin/pg_config}" + +N=24000 +psql_run "CREATE TABLE psio (ik int, bulky text) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('psio', stripe_row_limit => 1200, chunk_group_row_limit => 400);" +# Compressible payload so ANALYZE width is large while stored bytes stay +# modest. The covering projection is a second copy of the same columns, +# so rel->pages counts base plus projection. +psql_run "INSERT INTO psio SELECT ik, repeat('b', 900) FROM generate_series(1, $N) ik ORDER BY md5(ik::text);" +psql_run "SELECT pgcolumnar.add_projection('psio', 'byik', ARRAY['ik','bulky'], ARRAY['ik']);" +psql_run "ANALYZE psio;" + +explain_scan() { + # $1 = on|off for pgcolumnar.enable_projection_scan + # $2 = SQL + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -Atq \ + -c "SET max_parallel_workers_per_gather = 0;" \ + -c "SET pgcolumnar.enable_ungrouped_vector_agg = off;" \ + -c "SET pgcolumnar.enable_group_vectorization = off;" \ + -c "SET jit = off;" \ + -c "SET seq_page_cost = 1000;" \ + -c "SET cpu_tuple_cost = 0;" \ + -c "SET cpu_operator_cost = 0;" \ + -c "SET cpu_index_tuple_cost = 0;" \ + -c "SET pgcolumnar.enable_projection_scan = $1;" \ + -c "EXPLAIN (COSTS ON) $2" \ + | grep -v '^SET$' +} + +scan_cost_pair() { + echo "$1" | grep -F "Custom Scan (PgColumnarScan)" | head -1 \ + | grep -oE "cost=[0-9.]+\.\.[0-9.]+" | head -1 \ + | sed -E "s/cost=([0-9.]+)\\.\\.([0-9.]+)/\\1 \\2/" +} + +run_of() { + local pair start total + pair="$(scan_cost_pair "$1")" + start="${pair%% *}" + total="${pair##* }" + awk -v t="$total" -v s="$start" "BEGIN{ print t-s }" +} + +SQL="SELECT ik, bulky FROM psio WHERE ik BETWEEN 1 AND $N" +cover_plan="$(explain_scan on "$SQL")" +c_run="$(run_of "$cover_plan")" + +rel_pages="$(q "SELECT pg_relation_size('psio') / 8192.0")" +page_cost=1000 +base_io="$(awk -v p="$rel_pages" -v c="$page_cost" "BEGIN{ print p*c }")" +ratio="$(awk -v r="$c_run" -v i="$base_io" "BEGIN{ if (i<=0) print 0; else printf \"%.3f\", r/i }")" + +proj_bytes="$(q "SELECT coalesce(sum(rg.byte_length),0) FROM pgcolumnar.row_group rg JOIN pgcolumnar.projection p ON p.proj_storage_id = rg.storage_id JOIN pgcolumnar.storage s ON s.storage_id = p.storage_id WHERE s.relation_oid = 'psio'::regclass AND p.name = 'byik'")" +# Same per-row-group page rounding the cost model uses (COLUMNAR_PAGE_ROUND_UP). +proj_pages="$(q "SELECT coalesce(sum(ceil(rg.byte_length::numeric / (8192 - 24))),0) FROM pgcolumnar.row_group rg JOIN pgcolumnar.projection p ON p.proj_storage_id = rg.storage_id JOIN pgcolumnar.storage s ON s.storage_id = p.storage_id WHERE s.relation_oid = 'psio'::regclass AND p.name = 'byik'")" +rel_bytes="$(q "SELECT pg_relation_size('psio')")" +want_run="$(awk -v p="$proj_pages" -v c="$page_cost" "BEGIN{ print p*c }")" + +# Base scan with projection off, printed here for a reader. It is NOT the oracle +# the miss arm divides by; that one is read below, in the same catalog state as +# the arm. +base_plan="$(explain_scan off "$SQL")" +base_run="$(run_of "$base_plan")" + +echo "-- cover_run=$c_run rel_pages=$rel_pages base_io=$base_io ratio=$ratio" +echo "-- proj_bytes=$proj_bytes rel_bytes=$rel_bytes proj_pages=$proj_pages want_run=$want_run" +echo "-- base_run=$base_run" + +check "premise: the table holds every inserted row" \ + "$(q "SELECT count(*) FROM psio")" "$N" + +check "premise: a covering projection exists" \ + "$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = 'psio'::regclass AND name = 'byik'")" "1" + +check "premise: the plan uses the covering projection" \ + "$(echo "$cover_plan" | grep -c 'Columnar Projection: byik')" "1" + +check "premise: the covering scan has a positive run cost" \ + "$(awk -v c="$c_run" "BEGIN{ print (c>0) ? \"yes\" : \"no\" }")" "yes" + +# Without this, a pass could mean the projection filled the file and both +# formulae agree. Charging rel->pages and charging projection pages must +# not look the same. +check "premise: the covering projection occupies a minority of the relation" \ + "$(awk -v p="$proj_bytes" -v r="$rel_bytes" "BEGIN{ print (r>0 && p < r*0.7) ? \"minority\" : \"majority p=\" p \" r=\" r }")" \ + "minority" + +# Band around seq_page_cost * proj_pages (sel=1), not a ceiling against +# base_io. A half-priced projection still clears a 0.8 ceiling. +check "a covering projection is not priced from the base table's pages" \ + "$(awk -v g="$c_run" -v w="$want_run" "BEGIN{ if (w<=0) print \"no-want\"; else { d=(g>w?g-w:w-g)/w; print (d>0.05) ? \"off-band got=\" g \" want=\" w : \"proj-pages\" } }")" \ + "proj-pages" + +# Lookup failure must not make the covering path cheaper than the base file. +# Returning 1 page is essentially free; rel->pages cannot undercut the base +# scan. The path is still offered: choose_projection keys on the name. +psql_run "UPDATE pgcolumnar.projection p SET proj_storage_id = 0 FROM pgcolumnar.storage s WHERE p.storage_id = s.storage_id AND s.relation_oid = 'psio'::regclass AND p.name = 'byik' AND p.projection_id > 0;" + +check "premise: the projection row still exists after its storage id is cleared" \ + "$(q "SELECT count(*) FROM pgcolumnar.projection p JOIN pgcolumnar.storage s ON s.storage_id = p.storage_id WHERE s.relation_oid = 'psio'::regclass AND p.name = 'byik' AND p.proj_storage_id = 0")" \ + "1" + +# THE ORACLE AND THE ARM MUST BE READ IN THE SAME CATALOG STATE (#1155 review). +# The UPDATE above clears proj_storage_id, which is the SAME KEY #1180's sibling +# walk reads. So a base_run captured before it is priced with the projection's +# pages subtracted and a miss_run captured after it is priced from the whole +# file, and the ratio spans two different trees rather than measuring one. +# +# Composed with main carrying #1180, the old order read +# +# miss_run=41991.6 base_run=22000 miss_ratio=1.909 FAIL +# +# and re-reading the oracle here gives base_run=42000 and miss_ratio=1.000. The +# comment above used to claim this oracle was "independent of how rel->pages is +# computed", which is true of the covering arm -- whose want_run comes from the +# catalog and never mentions rel->pages -- and false of this one, whose oracle is +# a measured plan cost and moves with rel->pages like everything else. +# +# Before #1180 both states gave rel->pages = 42 and the ratio was exactly 1.000, +# so nothing here could have noticed. +miss_base_run="$(run_of "$(explain_scan off "$SQL")")" +miss_plan="$(explain_scan on "$SQL")" +miss_run="$(run_of "$miss_plan")" +miss_ratio="$(awk -v m="$miss_run" -v b="$miss_base_run" "BEGIN{ if (b<=0) print 0; else printf \"%.3f\", m/b }")" +echo "-- miss_run=$miss_run base_run=$miss_base_run miss_ratio=$miss_ratio" + +check "premise: a covering path is still offered when projection storage cannot be found" \ + "$(echo "$miss_plan" | grep -c 'Columnar Projection: byik')" "1" + +# Fallback to rel->pages must price like the base scan itself, not like one +# page. Compare to the measured base run (projection off), not to base_io +# from pg_relation_size, so #1180 cannot silently shift the denominator. +check "a covering projection whose storage cannot be found is not priced as one page" \ + "$(awk -v r="$miss_ratio" "BEGIN{ print (r+0 < 0.8 || r+0 > 1.25) ? \"moved miss_ratio=\" r : \"not-one-page\" }")" \ + "not-one-page" + +pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 1a4d7c82..15bdace7 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -124,6 +124,7 @@ behaviour, the source of that number is named. - [76. test_docs_upgrade_chain.py: the documented upgrade chain must be the one that ships](#76-test_docs_upgrade_chainpy-the-documented-upgrade-chain-must-be-the-one-that-ships) - [77. test_projection_parallel.py: a covering projection can be a parallel scan](#77-test_projection_parallelpy-a-covering-projection-can-be-a-parallel-scan) - [78. test_ttl_expire.py: the one function that deletes rows, tested twice](#78-test_ttl_expirepy-the-one-function-that-deletes-rows-tested-twice) +- [79. test_projection_scan_io.py: a covering projection is not priced from the base table's pages](#79-test_projection_scan_iopy-a-covering-projection-is-not-priced-from-the-base-tables-pages) ## 1. How to read a test in here @@ -6126,3 +6127,20 @@ The shell twin is `test/ttl_expire.sh`, 51 checks. The two halves share no code: different tables, different row counts, different retention windows, and the refusals asserted by SQLSTATE here (`55000` for no declared retention, `22023` for a non-positive interval) where the shell greps its message. + +## 79. test_projection_scan_io.py: a covering projection is not priced from the base table's pages + +Port of `projection_scan_io.sh`. A covering projection scan inherited the base +custom-scan I/O term, `seq_page_cost * rel->pages`. That page count is the whole +relation file: the base plus every projection stored beside it. The covering +path reads only the projection's own row groups. + +Public seam: `EXPLAIN` cost of a covering projection against `pg_relation_size` +of the table, with `seq_page_cost` raised and CPU terms zeroed so the run is +pages. A lookup that cannot find the projection's storage must fall back to +`rel->pages`, not one page. The shell twin uses `psio` / `byik` / 24000 rows; +this file uses `pciot` / `onck` / 36000 rows. Assertion names match. + +| test | what it holds | +| --- | --- | +| `test_projection_scan_io` | the table and covering projection exist; the plan uses that projection; the covering scan has a positive run cost; the projection occupies a minority of the relation; the covering run is not priced from the base table's pages; a covering path whose storage cannot be found is not priced as one page | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index e87bd122..8cc05b49 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -496,4 +496,8 @@ guard_tests 403 # of its hour and wrong for the next. `475 tests collected` over 49 cluster # files. `guard_tests` was re-derived in the same run and did NOT move, which # is the expected answer for a file that needs a database. -cluster_tests 475 +# 475 -> 476: this branch's own test (#1155). RE-DERIVED ON THE COMPOSED TREE +# after a THIRD rebase. This value has read 464, 468, 469 and now 476 as main +# went 463, 467, 468 and 475 under it; every one was correct for the main of +# its hour and none survived the next merge. `476 tests collected`. +cluster_tests 476 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 13932dd4..e34b3d1b 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -116,6 +116,7 @@ "projection_parallel", "projection_privilege", "projection_scan_cost", + "projection_scan_io", "projection_update", "projections", "scan_decode_cost", diff --git a/test/pytest/test_projection_scan_io.py b/test/pytest/test_projection_scan_io.py new file mode 100644 index 00000000..43c4b622 --- /dev/null +++ b/test/pytest/test_projection_scan_io.py @@ -0,0 +1,202 @@ +"""A covering projection must be priced from its own pages. + +PgColumnarSetRelPathlist offers a covering-projection path by scaling the +BASE scan's run cost. That run's I/O term is seq_page_cost * rel->pages, +the whole relation file. A covering projection has its own row groups. + +This file asserts the PLANNER ratio, not a runtime. Independent of +test/projection_scan_io.sh: same public seam (EXPLAIN cost of a covering +projection vs the relation's pages), own fixture, own observations. +Assertion names match the shell suite so the two can be compared by name, +not by importing each other. +""" + + +def _nodes(plan): + stack = [plan[0]["Plan"]] + while stack: + node = stack.pop(0) + yield node + stack.extend(node.get("Plans") or ()) + + +def _custom_scan(plan): + for node in _nodes(plan): + if node.get("Node Type") == "Custom Scan": + return node + return None + + +def _plan(conn, sql, projection_scan): + with conn.cursor() as cur: + cur.execute("SET max_parallel_workers_per_gather = 0") + cur.execute("SET pgcolumnar.enable_ungrouped_vector_agg = off") + cur.execute("SET pgcolumnar.enable_group_vectorization = off") + cur.execute("SET jit = off") + cur.execute("SET seq_page_cost = 1000") + cur.execute("SET cpu_tuple_cost = 0") + cur.execute("SET cpu_operator_cost = 0") + cur.execute("SET cpu_index_tuple_cost = 0") + cur.execute( + "SET pgcolumnar.enable_projection_scan = " + + ("on" if projection_scan else "off") + ) + cur.execute("EXPLAIN (FORMAT JSON, COSTS ON) " + sql) + return cur.fetchone()[0] + + +def test_projection_scan_io(pgc_conn, expect): + n = 36000 + with pgc_conn.cursor() as cur: + cur.execute( + "CREATE TABLE pciot (ck int, wide text) USING pgcolumnar" + ) + cur.execute( + "SELECT pgcolumnar.set_options('pciot', stripe_row_limit => 1800, " + "chunk_group_row_limit => 600)" + ) + # Compressible payload, independent of the shell twin: different + # table, N, stripe, column names, and repeat length. + cur.execute( + f"INSERT INTO pciot SELECT ck, repeat('w', 1100) " + f"FROM generate_series(1, {n}) ck ORDER BY md5((ck + 3)::text)" + ) + cur.execute( + "SELECT pgcolumnar.add_projection('pciot', 'onck', " + "ARRAY['ck','wide'], ARRAY['ck'])" + ) + cur.execute("ANALYZE pciot") + cur.execute("SELECT count(*) FROM pciot") + expect.num(cur.fetchone()[0], n, "premise: the table holds every inserted row") + cur.execute( + "SELECT count(*) FROM pgcolumnar.projection_declaration " + "WHERE rel = 'pciot'::regclass AND name = 'onck'" + ) + expect.num(cur.fetchone()[0], 1, "premise: a covering projection exists") + + sql = f"SELECT ck, wide FROM pciot WHERE ck BETWEEN 1 AND {n}" + cover = _plan(pgc_conn, sql, True) + node = _custom_scan(cover) + + expect.text( + (node or {}).get("Columnar Projection") or "none", + "onck", + "premise: the plan uses the covering projection", + ) + + c_run = node["Total Cost"] - node["Startup Cost"] + expect.text( + "yes" if c_run > 0 else "no", + "yes", + "premise: the covering scan has a positive run cost", + ) + + with pgc_conn.cursor() as cur: + cur.execute("SELECT pg_relation_size('pciot')") + rel_bytes = cur.fetchone()[0] + cur.execute( + "SELECT coalesce(sum(rg.byte_length),0) " + "FROM pgcolumnar.row_group rg " + "JOIN pgcolumnar.projection p ON p.proj_storage_id = rg.storage_id " + "JOIN pgcolumnar.storage s ON s.storage_id = p.storage_id " + "WHERE s.relation_oid = 'pciot'::regclass AND p.name = 'onck'" + ) + proj_bytes = cur.fetchone()[0] + # Same per-row-group page rounding the cost model uses. + cur.execute( + "SELECT coalesce(sum(ceil(rg.byte_length::numeric / (8192 - 24))),0) " + "FROM pgcolumnar.row_group rg " + "JOIN pgcolumnar.projection p ON p.proj_storage_id = rg.storage_id " + "JOIN pgcolumnar.storage s ON s.storage_id = p.storage_id " + "WHERE s.relation_oid = 'pciot'::regclass AND p.name = 'onck'" + ) + proj_pages = float(cur.fetchone()[0]) + + rel_pages = rel_bytes / 8192.0 + base_io = rel_pages * 1000.0 + ratio = (c_run / base_io) if base_io > 0 else 0.0 + want_run = proj_pages * 1000.0 + base = _plan(pgc_conn, sql, False) + base_node = _custom_scan(base) + base_run = base_node["Total Cost"] - base_node["Startup Cost"] + print( + f"-- cover_run={c_run} rel_pages={rel_pages} " + f"base_io={base_io} ratio={ratio:.3f}" + ) + print( + f"-- proj_bytes={proj_bytes} rel_bytes={rel_bytes} " + f"proj_pages={proj_pages} want_run={want_run}" + ) + print(f"-- base_run={base_run}") + + expect.text( + "minority" if rel_bytes > 0 and proj_bytes < rel_bytes * 0.7 + else f"majority proj_bytes={proj_bytes} rel_bytes={rel_bytes}", + "minority", + "premise: the covering projection occupies a minority of the relation", + ) + # Band around seq_page_cost * proj_pages, not a ceiling against base_io. + delta = abs(c_run - want_run) / want_run if want_run > 0 else 1.0 + expect.text( + f"off-band got={c_run} want={want_run}" + if delta > 0.05 else "proj-pages", + "proj-pages", + "a covering projection is not priced from the base table's pages", + ) + + # Lookup failure must not make the covering path cheaper than the base + # file. Returning 1 page is essentially free; rel->pages cannot undercut + # the base scan. Own table (pciot / onck), not the shell twin's. + with pgc_conn.cursor() as cur: + cur.execute( + "UPDATE pgcolumnar.projection p SET proj_storage_id = 0 " + "FROM pgcolumnar.storage s " + "WHERE p.storage_id = s.storage_id " + "AND s.relation_oid = 'pciot'::regclass " + "AND p.name = 'onck' AND p.projection_id > 0" + ) + cur.execute( + "SELECT count(*) FROM pgcolumnar.projection p " + "JOIN pgcolumnar.storage s ON s.storage_id = p.storage_id " + "WHERE s.relation_oid = 'pciot'::regclass " + "AND p.name = 'onck' AND p.proj_storage_id = 0" + ) + expect.num( + cur.fetchone()[0], + 1, + "premise: the projection row still exists after its storage id is cleared", + ) + + miss = _plan(pgc_conn, sql, True) + miss_node = _custom_scan(miss) + expect.text( + (miss_node or {}).get("Columnar Projection") or "none", + "onck", + "premise: a covering path is still offered when projection storage cannot be found", + ) + # THE ORACLE AND THE ARM MUST BE READ IN THE SAME CATALOG STATE (#1155 review). + # The UPDATE above clears `proj_storage_id`, which is the SAME KEY #1180's + # sibling-pages walk reads. A `base_run` captured before it is priced with the + # projection's pages subtracted; a `miss_run` captured after it is priced from + # the whole file. The ratio then spans two different trees rather than + # measuring one. + # + # Composed with main carrying #1180 the old order read + # `miss_run=41991.6 base_run=22000 miss_ratio=1.909` and failed; re-reading + # the oracle here gives `base_run=42000` and `miss_ratio=1.000`. + # + # Before #1180 both states gave the same `rel->pages` and the ratio was + # exactly 1.000, so nothing here could have noticed. + miss_base = _custom_scan(_plan(pgc_conn, sql, False)) + miss_base_run = miss_base["Total Cost"] - miss_base["Startup Cost"] + miss_run = miss_node["Total Cost"] - miss_node["Startup Cost"] + miss_ratio = (miss_run / miss_base_run) if miss_base_run > 0 else 0.0 + print(f"-- miss_run={miss_run} base_run={miss_base_run} miss_ratio={miss_ratio:.3f}") + # Compare to the measured base run (projection off), not base_io from + # pg_relation_size, so #1180 cannot silently shift the denominator. + expect.text( + f"moved miss_ratio={miss_ratio:.3f} (miss_run={miss_run}, base_run={miss_base_run})" + if miss_ratio < 0.8 or miss_ratio > 1.25 else "not-one-page", + "not-one-page", + "a covering projection whose storage cannot be found is not priced as one page", + ) diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 00363568..2b3fb129 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -259,6 +259,7 @@ SUITES=( projection_rename_restore projection_rewrite projection_scan_cost + projection_scan_io projection_update projections pushdown_report