Merge pull request #50 from ProxySQL/feat/postgresql-compat-harness

[codex] PostgreSQL compatibility harness foundation

Author: renecannao

.github/workflows/ci.yml

@@ -5,9 +5,12 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  schedule:
+    - cron: '17 3 * * 1'
 
 jobs:
   build-and-test:
+    if: github.event_name != 'schedule'
     strategy:
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
@@ -22,6 +25,7 @@ jobs:
         run: make -f Makefile clean && make -f Makefile all
 
   macos:
+    if: github.event_name != 'schedule'
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
@@ -38,6 +42,7 @@ jobs:
         run: make -f Makefile clean && make -f Makefile all MYSQL_CFLAGS="-I/opt/homebrew/opt/mysql-client/include"
 
   benchmark:
+    if: github.event_name != 'schedule'
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
@@ -58,6 +63,7 @@ jobs:
           path: benchmark_results.json
 
   corpus-test:
+    if: github.event_name != 'schedule'
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
@@ -96,3 +102,43 @@ jobs:
           grep -ohP '"((?:SELECT|INSERT|UPDATE|DELETE|SET|CREATE|ALTER|DROP|EXPLAIN|WITH)[^"]*)"' \
               /tmp/sqlparser-rs/tests/sqlparser_postgres.rs 2>/dev/null | \
               sed 's/^"//' | sed 's/"$//' | sed 's/\\"/"/g' | ./corpus_test pgsql
+
+  pg-compat:
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      PG_COMPAT_CACHE: /tmp/parsersql-pg-compat
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache PostgreSQL compatibility sources
+        uses: actions/cache@v4
+        with:
+          path: /tmp/parsersql-pg-compat
+          key: pg-compat-${{ runner.os }}-${{ hashFiles('tests/pg_compat/upstream_pins.json') }}
+          restore-keys: |
+            pg-compat-${{ runner.os }}-
+
+      - name: Test PostgreSQL compatibility gate
+        run: make -f Makefile test-pg-compat
+
+  pg-compat-full:
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    env:
+      PG_COMPAT_CACHE: /tmp/parsersql-pg-compat
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache PostgreSQL compatibility sources
+        uses: actions/cache@v4
+        with:
+          path: /tmp/parsersql-pg-compat
+          key: pg-compat-${{ runner.os }}-${{ hashFiles('tests/pg_compat/upstream_pins.json') }}
+          restore-keys: |
+            pg-compat-${{ runner.os }}-
+
+      - name: Validate full PostgreSQL compatibility baseline
+        run: make -f Makefile pg-compat

.gitignore

@@ -31,6 +31,9 @@
 *.out
 *.app
 
+# Local git worktrees
+.worktrees/
+
 # Build artifacts
 libsqlparser.a
 sqlengine
@@ -39,4 +42,6 @@ run_tests_debug
 corpus_test
 run_bench
 run_bench_compare
+pg_compat_17
+pg_compat_18
 bench/sqlparser_rs_bench/target/

Makefile

@@ -7,6 +7,7 @@ MYSQL_CFLAGS = $(shell mysql_config --cflags 2>/dev/null)
 MYSQL_LIBS = $(shell mysql_config --libs 2>/dev/null)
 PG_CFLAGS = -I$(shell pg_config --includedir 2>/dev/null || echo /usr/include/postgresql)
 PG_LIBS = -L$(shell pg_config --libdir 2>/dev/null || echo /usr/lib/x86_64-linux-gnu) -lpq
+PG_COMPAT_CACHE ?= /tmp/parsersql-pg-compat
 
 PROJECT_ROOT = .
 SRC_DIR = $(PROJECT_ROOT)/src/sql_parser
@@ -122,12 +123,26 @@ ENGINE_STRESS_TARGET = engine_stress_test
 MYSQL_SERVER_SRC = $(PROJECT_ROOT)/tools/mysql_server.cpp
 MYSQL_SERVER_TARGET = mysql_server
 
-.PHONY: all lib test test-sqlengine test-sqlengine-in-memory test-sqlengine-single test-sqlengine-sharded bench bench-compare bench-distributed build-corpus-test build-sqlengine engine-stress mysql-server clean
+.PHONY: all lib test test-sqlengine test-sqlengine-in-memory test-sqlengine-single test-sqlengine-sharded bench bench-compare bench-distributed build-corpus-test build-sqlengine build-pg-compat pg-compat pg-compat-refresh test-pg-compat engine-stress mysql-server clean
 
 build-corpus-test: $(CORPUS_TEST_TARGET)
 
 build-sqlengine: $(SQLENGINE_TARGET)
 
+build-pg-compat: lib
+	PG_COMPAT_CACHE=$(PG_COMPAT_CACHE) ./scripts/pg_compat/fetch_libpg_query.sh
+	python3 ./scripts/pg_compat/run_compat.py build --cache $(PG_COMPAT_CACHE)
+
+test-pg-compat: build-pg-compat
+	PG_COMPAT_RUNNER=$(PG_COMPAT_CACHE)/bin/pg_compat-18 python3 -m unittest discover -s tests/pg_compat -p 'test_*.py' -v
+	python3 ./scripts/pg_compat/run_compat.py test --cache $(PG_COMPAT_CACHE)
+
+pg-compat: build-pg-compat
+	python3 ./scripts/pg_compat/run_compat.py full --cache $(PG_COMPAT_CACHE)
+
+pg-compat-refresh: build-pg-compat
+	python3 ./scripts/pg_compat/run_compat.py refresh --cache $(PG_COMPAT_CACHE)
+
 all: lib test
 
 lib: $(LIB_TARGET)

README.md

@@ -64,6 +64,26 @@ make build-sqlengine    # Interactive SQL CLI
 make build-corpus-test  # Corpus validation harness
 ```
 
+### PostgreSQL compatibility harness
+
+ParserSQL uses `libpg_query` as the PostgreSQL syntax oracle for compatibility checks. The harness pins `libpg_query` `17-latest` and `18-latest`, extracts accepted PostgreSQL regression statements, and compares ParserSQL behavior against the committed PostgreSQL 18 baseline in [`docs/compatibility/postgresql-18.md`](docs/compatibility/postgresql-18.md).
+
+Compatibility results separate full parser parity from classification coverage:
+
+- `DEEP_SUPPORTED` means ParserSQL accepted the statement with the expected statement type and a full AST.
+- `CLASSIFIED_ONLY` means ParserSQL recognized the top-level statement class, but did not produce full deep parser parity for that SQL.
+- `PARTIAL`, `ERROR`, `TRAILING_INPUT`, and `TYPE_MISMATCH` are tracked separately in the committed baseline.
+
+The report includes the full PostgreSQL 18 backlog, the PG17-to-PG18 release delta, statement routing coverage, and reviewed structural grammar/keyword changes.
+
+```bash
+make test-pg-compat     # Build the pinned oracle runner, run harness unit tests, and replay committed CI cases
+make pg-compat          # Run the full pinned PG17/PG18 comparison against the committed baseline
+make pg-compat-refresh  # Regenerate expected_results.jsonl, ci_cases.jsonl, and docs/compatibility/postgresql-18.md
+```
+
+`PG_COMPAT_CACHE` controls where external `libpg_query` and PostgreSQL source checkouts are cached. It defaults to `/tmp/parsersql-pg-compat`.
+
 ### Interactive CLI (`sqlengine`)
 
 ```bash

docs/benchmarks/REPRODUCING.md

@@ -9,7 +9,7 @@ Step-by-step instructions to reproduce all comparison benchmarks from scratch on
 ```bash
 # Required tools
 sudo apt-get update
-sudo apt-get install -y build-essential git curl
+sudo apt-get install -y build-essential git curl tar bzip2
 
 # Rust (for sqlparser-rs benchmark)
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
@@ -19,6 +19,9 @@ source ~/.cargo/env
 g++ --version    # need GCC 8+ (C++17 support)
 cargo --version  # need Rust 1.70+
 git --version
+curl --version
+tar --version
+python3 --version
 ```
 
 ---
@@ -76,6 +79,44 @@ ls -la third_party/libpg_query/libpg_query.a
 
 ---
 
+## PostgreSQL Compatibility Harness
+
+The PostgreSQL compatibility harness uses pinned `libpg_query` builds as the syntax oracle. It compares ParserSQL against PostgreSQL 17 and PostgreSQL 18 regression SQL, writes a committed PostgreSQL 18 baseline, and reports the PG17-to-PG18 syntax delta in `docs/compatibility/postgresql-18.md`.
+
+Required external tools:
+
+```text
+git
+curl
+tar with bzip2 support
+python3
+C/C++ compiler
+make
+```
+
+`PG_COMPAT_CACHE` controls where external sources and built oracle runners are cached. It defaults to `/tmp/parsersql-pg-compat`.
+
+```bash
+# Fast deterministic gate: unit tests plus committed CI cases
+make test-pg-compat
+
+# Full pinned PG17/PG18 comparison against committed expected_results.jsonl
+make pg-compat
+
+# Refresh committed artifacts after intentionally updating pins or mappings
+make pg-compat-refresh
+```
+
+The harness result model distinguishes parser depth from statement routing:
+
+- `DEEP_SUPPORTED` means ParserSQL accepted the SQL with the expected statement type and a full AST.
+- `CLASSIFIED_ONLY` means ParserSQL recognized the top-level statement class, but did not reach full parser parity for that SQL.
+- `PARTIAL`, `ERROR`, `TRAILING_INPUT`, and `TYPE_MISMATCH` are committed baseline outcomes that must not regress silently.
+
+The refresh target updates `tests/pg_compat/expected_results.jsonl`, `tests/pg_compat/ci_cases.jsonl`, and `docs/compatibility/postgresql-18.md`.
+
+---
+
 ## Step 4: Build the comparison benchmark
 
 ```bash