diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 09ad199c8e..e25964dd08 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -7,4 +7,5 @@ dev/release/rat_exclude_files.txt filtered_rat.txt AGENTS.md fuzz/fuzz_seeds/* +fuzz/vendor/* .github/PULL_REQUEST_TEMPLATE.md diff --git a/docs/fuzzing.md b/docs/fuzzing.md index e72d24ad2c..cd5054903b 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -37,7 +37,9 @@ rendered statement no longer parses. `fuzz_duckdb_accepts` parses the input with DuckDB's own parser and fails when DuckDB accepts a `SELECT` that `DuckDbDialect` rejects. -The three oracle targets compile their engine from source, so each builds only with its feature, `sqlite`, `postgres` or `duckdb`, as in `cargo +nightly fuzz run --features sqlite fuzz_sqlite_accepts fuzz_seeds`. `cargo +nightly fuzz build --all-features` builds every target. +`fuzz_clickhouse_lexes` tokenizes the input with ClickHouse's own lexer, from the [`clickhouse-lexer-sys`](https://crates.io/crates/clickhouse-lexer-sys) crate, and fails when it reaches the end without an error token but `ClickHouseDialect` fails to tokenize. ClickHouse has no standalone parser, so the check stops at the tokenizer; a lone `$`, which lexes but never parses, counts as rejected. + +The four oracle targets compile their engine from source, so each builds only with its feature, `sqlite`, `postgres`, `duckdb` or `clickhouse`, as in `cargo +nightly fuzz run --features sqlite fuzz_sqlite_accepts fuzz_seeds`. `cargo +nightly fuzz build --all-features` builds every target. `fuzz_stage_cost` times tokenizing, parsing and printing the input with every dialect and fails when parsing takes more than 50 times as long as tokenizing, or printing more than 50 times as long as parsing, once the slower stage passes 10 ms. It reports superlinear paths whose cost stays far below the fuzzer's timeout. diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 654e48037a..72a589b624 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -29,6 +29,7 @@ libfuzzer-sys = "0.4" libsqlite3-sys = { version = "0.38", features = ["bundled"], optional = true } pg_query = { version = "6.2", optional = true } libduckdb-sys = { version = "1.10505.0", features = ["bundled", "json"], optional = true } +clickhouse-lexer-sys = { version = "0.1", optional = true } serde_json = { version = "1", optional = true } sqlparser = { path = ".." } @@ -37,6 +38,7 @@ sqlparser = { path = ".." } sqlite = ["dep:libsqlite3-sys"] postgres = ["dep:pg_query"] duckdb = ["dep:libduckdb-sys", "dep:serde_json"] +clickhouse = ["dep:clickhouse-lexer-sys"] # Prevent this from interfering with workspaces [workspace] @@ -80,6 +82,14 @@ test = false doc = false bench = false +[[bin]] +name = "fuzz_clickhouse_lexes" +path = "fuzz_targets/fuzz_clickhouse_lexes.rs" +required-features = ["clickhouse"] +test = false +doc = false +bench = false + [[bin]] name = "fuzz_stage_cost" path = "fuzz_targets/fuzz_stage_cost.rs" diff --git a/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs b/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs new file mode 100644 index 0000000000..44d68cc035 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![no_main] + +use clickhouse_lexer_sys::{Lexer, TokenKind, UPSTREAM_COMMIT}; +use libfuzzer_sys::fuzz_target; +use sqlparser::dialect::ClickHouseDialect; +use sqlparser::tokenizer::Tokenizer; + +/// Whether ClickHouse's lexer reads `sql` to the end without an error token or a lone `$`. +/// +/// A lone `$` lexes as `DollarSign`, which only the JSONPath parser consumes, so no +/// statement holding one parses. +fn clickhouse_lexes(sql: &str) -> bool { + Lexer::new(sql).all(|token| !token.kind.is_error() && token.kind != TokenKind::DollarSign) +} + +fuzz_target!(|sql: &str| { + if !clickhouse_lexes(sql) { + return; + } + if let Err(err) = Tokenizer::new(&ClickHouseDialect {}, sql).tokenize() { + panic!( + "ClickHouse lexer at {UPSTREAM_COMMIT} accepts SQL the ClickHouse dialect tokenizer rejects\n sql: {sql:?}\n error: {err}" + ) + } +});