From 7fa5a5390b08a5a68aa4a3efe2647a96ab15109b Mon Sep 17 00:00:00 2001 From: s5dsn-eqee Date: Fri, 16 Oct 2026 18:25:42 +0200 Subject: [PATCH 1/2] ClickHouse: Add fuzz target comparing the tokenizer with ClickHouse's lexer --- dev/release/rat_exclude_files.txt | 1 + docs/fuzzing.md | 4 +- fuzz/Cargo.toml | 12 + fuzz/build.rs | 34 + fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs | 80 +++ fuzz/vendor/clickhouse-lexer/COMMIT | 1 + fuzz/vendor/clickhouse-lexer/LICENSE | 203 ++++++ .../vendor/clickhouse-lexer/Parsers/Lexer.cpp | 650 ++++++++++++++++++ fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h | 150 ++++ .../Parsers/LexerStandalone.h | 180 +++++ .../Parsers/clickhouse_lexer.h | 22 + fuzz/vendor/clickhouse-lexer/README.md | 14 + 12 files changed, 1350 insertions(+), 1 deletion(-) create mode 100644 fuzz/build.rs create mode 100644 fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs create mode 100644 fuzz/vendor/clickhouse-lexer/COMMIT create mode 100644 fuzz/vendor/clickhouse-lexer/LICENSE create mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp create mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h create mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h create mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h create mode 100644 fuzz/vendor/clickhouse-lexer/README.md diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 09ad199c8e..e25964dd08 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -7,4 +7,5 @@ dev/release/rat_exclude_files.txt filtered_rat.txt AGENTS.md fuzz/fuzz_seeds/* +fuzz/vendor/* .github/PULL_REQUEST_TEMPLATE.md diff --git a/docs/fuzzing.md b/docs/fuzzing.md index e72d24ad2c..861c4dc93b 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -37,7 +37,9 @@ rendered statement no longer parses. `fuzz_duckdb_accepts` parses the input with DuckDB's own parser and fails when DuckDB accepts a `SELECT` that `DuckDbDialect` rejects. -The three oracle targets compile their engine from source, so each builds only with its feature, `sqlite`, `postgres` or `duckdb`, as in `cargo +nightly fuzz run --features sqlite fuzz_sqlite_accepts fuzz_seeds`. `cargo +nightly fuzz build --all-features` builds every target. +`fuzz_clickhouse_lexes` tokenizes the input with ClickHouse's own lexer, vendored under `fuzz/vendor/clickhouse-lexer/`, and fails when it reaches the end without an error token but `ClickHouseDialect` fails to tokenize. ClickHouse has no standalone parser, so the check stops at the tokenizer. + +The four oracle targets compile their engine from source, so each builds only with its feature, `sqlite`, `postgres`, `duckdb` or `clickhouse`, as in `cargo +nightly fuzz run --features sqlite fuzz_sqlite_accepts fuzz_seeds`. `cargo +nightly fuzz build --all-features` builds every target. `fuzz_stage_cost` times tokenizing, parsing and printing the input with every dialect and fails when parsing takes more than 50 times as long as tokenizing, or printing more than 50 times as long as parsing, once the slower stage passes 10 ms. It reports superlinear paths whose cost stays far below the fuzzer's timeout. diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 654e48037a..dabd6090bb 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -32,11 +32,15 @@ libduckdb-sys = { version = "1.10505.0", features = ["bundled", "json"], optiona serde_json = { version = "1", optional = true } sqlparser = { path = ".." } +[build-dependencies] +cc = { version = "1", optional = true } + # The oracle targets build their engines from source, so each compiles only when its feature is on. [features] sqlite = ["dep:libsqlite3-sys"] postgres = ["dep:pg_query"] duckdb = ["dep:libduckdb-sys", "dep:serde_json"] +clickhouse = ["dep:cc"] # Prevent this from interfering with workspaces [workspace] @@ -80,6 +84,14 @@ test = false doc = false bench = false +[[bin]] +name = "fuzz_clickhouse_lexes" +path = "fuzz_targets/fuzz_clickhouse_lexes.rs" +required-features = ["clickhouse"] +test = false +doc = false +bench = false + [[bin]] name = "fuzz_stage_cost" path = "fuzz_targets/fuzz_stage_cost.rs" diff --git a/fuzz/build.rs b/fuzz/build.rs new file mode 100644 index 0000000000..9b9feff18d --- /dev/null +++ b/fuzz/build.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +fn main() { + #[cfg(feature = "clickhouse")] + { + let dir = "vendor/clickhouse-lexer"; + println!("cargo:rerun-if-changed={dir}"); + cc::Build::new() + .cpp(true) + .std("c++20") + .include(dir) + .file(format!("{dir}/Parsers/Lexer.cpp")) + .define("LEXER_STANDALONE_BUILD", None) + .flag("-fno-exceptions") + .flag("-fno-rtti") + .warnings(false) + .compile("clickhouse_lexer"); + } +} diff --git a/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs b/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs new file mode 100644 index 0000000000..a810166c1c --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use sqlparser::dialect::ClickHouseDialect; +use sqlparser::tokenizer::Tokenizer; +use std::ffi::{c_char, c_int, c_uchar, c_void}; + +const COMMIT: &str = include_str!("../vendor/clickhouse-lexer/COMMIT"); + +// C ABI of the vendored lexer, see fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h. +extern "C" { + static clickhouse_lexer_size: usize; + fn clickhouse_lexer_create( + ptr: *mut c_void, + begin: *const c_char, + end: *const c_char, + max_query_size: usize, + ); + fn clickhouse_lexer_next_token( + ptr: *mut c_void, + out_token_begin: *mut *const c_char, + out_token_end: *mut *const c_char, + ) -> c_uchar; + fn clickhouse_lexer_token_is_error(token: c_uchar) -> c_int; + fn clickhouse_lexer_token_is_end(token: c_uchar) -> c_int; +} + +/// Whether ClickHouse's lexer reads `sql` to the end without an error token. +fn clickhouse_lexes(sql: &str) -> bool { + // SAFETY: the symbol is a plain `size_t` the C++ side initializes statically. + let size = unsafe { clickhouse_lexer_size }; + // u64 slots keep the pointer fields of the C++ Lexer aligned. + let mut lexer = vec![0u64; size.div_ceil(8)]; + let ptr = lexer.as_mut_ptr().cast::(); + let begin = sql.as_ptr().cast::(); + // SAFETY: `lexer` holds `clickhouse_lexer_size` bytes and `sql` outlives every call below. + unsafe { + clickhouse_lexer_create(ptr, begin, begin.add(sql.len()), 0); + loop { + let mut token_begin = std::ptr::null(); + let mut token_end = std::ptr::null(); + let token = clickhouse_lexer_next_token(ptr, &mut token_begin, &mut token_end); + if clickhouse_lexer_token_is_error(token) != 0 { + return false; + } + if clickhouse_lexer_token_is_end(token) != 0 { + return true; + } + } + } +} + +fuzz_target!(|sql: &str| { + if !clickhouse_lexes(sql) { + return; + } + if let Err(err) = Tokenizer::new(&ClickHouseDialect {}, sql).tokenize() { + panic!( + "ClickHouse lexer at {} accepts SQL the ClickHouse dialect tokenizer rejects\n sql: {sql:?}\n error: {err}", + COMMIT.trim() + ) + } +}); diff --git a/fuzz/vendor/clickhouse-lexer/COMMIT b/fuzz/vendor/clickhouse-lexer/COMMIT new file mode 100644 index 0000000000..70ee81fc65 --- /dev/null +++ b/fuzz/vendor/clickhouse-lexer/COMMIT @@ -0,0 +1 @@ +55c2563639c7ab7d90aa85f52351ec35c936bae7 diff --git a/fuzz/vendor/clickhouse-lexer/LICENSE b/fuzz/vendor/clickhouse-lexer/LICENSE new file mode 100644 index 0000000000..f1da7e1a39 --- /dev/null +++ b/fuzz/vendor/clickhouse-lexer/LICENSE @@ -0,0 +1,203 @@ +Copyright 2016-2026 ClickHouse, Inc. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2026 ClickHouse, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp b/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp new file mode 100644 index 0000000000..e2f3a41374 --- /dev/null +++ b/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp @@ -0,0 +1,650 @@ +// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/Lexer.cpp, +// which mirrors src/Parsers/Lexer.cpp of https://github.com/ClickHouse/ClickHouse at the +// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. + +#if !defined(LEXER_STANDALONE_BUILD) + +#include +#include +#include +#include +#include + +#else /// This allows building Lexer without any dependencies or includes for WebAssembly or Emscripten. + +#include +#include + +#endif + + +namespace DB +{ + +namespace +{ + +/// This must be consistent with functions in ReadHelpers.h +template +Token quotedString(const char *& pos, const char * const token_begin, const char * const end, + TokenType success_token, TokenType error_token) +{ + ++pos; + while (true) + { + pos = find_first_symbols(pos, end); + if (pos >= end) + return Token(error_token, token_begin, end); + + if (*pos == quote) + { + ++pos; + if (pos < end && *pos == quote) + { + ++pos; + continue; + } + return Token(success_token, token_begin, pos); + } + + if (*pos == '\\') + { + ++pos; + if (pos >= end) + return Token(error_token, token_begin, end); + ++pos; + continue; + } + + chassert(false); + } +} + +Token quotedStringWithUnicodeQuotes(const char *& pos, const char * const token_begin, const char * const end, + char expected_end_byte, TokenType success_token, TokenType error_token) +{ + /// ‘: e2 80 98 + /// ’: e2 80 99 + /// “: e2 80 9c + /// ”: e2 80 9d + + while (true) + { + pos = find_first_symbols<'\xE2'>(pos, end); + if (pos + 2 >= end) + return Token(error_token, token_begin, end); + + if (pos[0] == '\xE2' && pos[1] == '\x80' && pos[2] == expected_end_byte) + { + pos += 3; + return Token(success_token, token_begin, pos); + } + + ++pos; + } +} + +Token quotedHexOrBinString(const char *& pos, const char * const token_begin, const char * const end) +{ + constexpr char quote = '\''; + + chassert(pos[1] == quote); + + bool hex = (*pos == 'x' || *pos == 'X'); + + pos += 2; + + if (hex) + { + while (pos < end && isHexDigit(*pos)) + ++pos; + } + else + { + pos = find_first_not_symbols<'0', '1'>(pos, end); + } + + if (pos >= end || *pos != quote) + { + pos = end; + return Token(TokenType::ErrorSingleQuoteIsNotClosed, token_begin, end); + } + + ++pos; + return Token(TokenType::StringLiteral, token_begin, pos); +} + +} + + +Token Lexer::nextToken() +{ + Token res = nextTokenImpl(); + if (begin && max_query_size && res.end > begin + max_query_size) + res.type = TokenType::ErrorMaxQuerySizeExceeded; + if (res.isSignificant()) + prev_significant_token_type = res.type; + return res; +} + + +Token Lexer::nextTokenImpl() +{ + if (pos >= end) + return Token(TokenType::EndOfStream, end, end); + + const char * const token_begin = pos; + + auto comment_until_end_of_line = [&]() mutable + { + pos = find_first_symbols<'\n'>(pos, end); /// This means that newline in single-line comment cannot be escaped. + return Token(TokenType::Comment, token_begin, pos); + }; + + switch (*pos) + { + case ' ': + case '\t': + case '\n': + case '\r': + case '\f': + case '\v': + { + ++pos; + while (pos < end && isWhitespaceASCII(*pos)) + ++pos; + return Token(TokenType::Whitespace, token_begin, pos); + } + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + /// The task is not to parse a number or check correctness, but only to skip it. + + /// Disambiguation: if previous token was dot, then we could parse only simple integer, + /// for chained tuple access operators (x.1.1) to work. + // Otherwise it will be tokenized as x . 1.1, not as x . 1 . 1 + if (prev_significant_token_type == TokenType::Dot) + { + ++pos; + while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(false, false, pos, end))) + ++pos; + } + else + { + bool start_of_block = false; + /// 0x, 0b + bool hex = false; + if (pos + 2 < end && *pos == '0' && (pos[1] == 'x' || pos[1] == 'b' || pos[1] == 'X' || pos[1] == 'B')) + { + bool is_valid = false; + if (pos[1] == 'x' || pos[1] == 'X') + { + if (isHexDigit(pos[2])) + { + hex = true; + is_valid = true; // hex + } + } + else if (pos[2] == '0' || pos[2] == '1') + is_valid = true; // bin + if (is_valid) + { + pos += 2; + start_of_block = true; + } + else + ++pos; // consume the leading zero - could be an identifier + } + else + ++pos; + + while (pos < end && ((hex ? isHexDigit(*pos) : isNumericASCII(*pos)) || isNumberSeparator(start_of_block, hex, pos, end))) + { + ++pos; + start_of_block = false; + } + + /// decimal point + if (pos < end && *pos == '.') + { + start_of_block = true; + ++pos; + while (pos < end && ((hex ? isHexDigit(*pos) : isNumericASCII(*pos)) || isNumberSeparator(start_of_block, hex, pos, end))) + { + ++pos; + start_of_block = false; + } + } + + /// exponentiation (base 10 or base 2) + if (pos + 1 < end && (hex ? (*pos == 'p' || *pos == 'P') : (*pos == 'e' || *pos == 'E'))) + { + start_of_block = true; + ++pos; + + /// sign of exponent. It is always decimal. + if (pos + 1 < end && (*pos == '-' || *pos == '+')) + ++pos; + + while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(start_of_block, false, pos, end))) + { + ++pos; + start_of_block = false; + } + } + } + + /// Try to parse it to a identifier(1identifier_name), otherwise it return ErrorWrongNumber + if (pos < end && isWordCharASCII(*pos)) + { + ++pos; + while (pos < end && isWordCharASCII(*pos)) + ++pos; + + for (const char * iterator = token_begin; iterator < pos; ++iterator) + { + if (!isWordCharASCII(*iterator) && *iterator != '$') + return Token(TokenType::ErrorWrongNumber, token_begin, pos); + } + + return Token(TokenType::BareWord, token_begin, pos); + } + + return Token(TokenType::Number, token_begin, pos); + } + + case '\'': + return quotedString<'\''>(pos, token_begin, end, TokenType::StringLiteral, TokenType::ErrorSingleQuoteIsNotClosed); + case '"': + return quotedString<'"'>(pos, token_begin, end, TokenType::QuotedIdentifier, TokenType::ErrorDoubleQuoteIsNotClosed); + case '`': + return quotedString<'`'>(pos, token_begin, end, TokenType::QuotedIdentifier, TokenType::ErrorBackQuoteIsNotClosed); + + case '(': + return Token(TokenType::OpeningRoundBracket, token_begin, ++pos); + case ')': + return Token(TokenType::ClosingRoundBracket, token_begin, ++pos); + case '[': + return Token(TokenType::OpeningSquareBracket, token_begin, ++pos); + case ']': + return Token(TokenType::ClosingSquareBracket, token_begin, ++pos); + case '{': + return Token(TokenType::OpeningCurlyBrace, token_begin, ++pos); + case '}': + return Token(TokenType::ClosingCurlyBrace, token_begin, ++pos); + case ',': + return Token(TokenType::Comma, token_begin, ++pos); + case ';': + return Token(TokenType::Semicolon, token_begin, ++pos); + + case '.': /// qualifier, tuple access operator or start of floating point number + { + /// Just after identifier or complex expression or number (for chained tuple access like x.1.1 to work properly). + if (pos > begin + && (!(pos + 1 < end && isNumericASCII(pos[1])) + || prev_significant_token_type == TokenType::ClosingRoundBracket + || prev_significant_token_type == TokenType::ClosingSquareBracket + || prev_significant_token_type == TokenType::BareWord + || prev_significant_token_type == TokenType::QuotedIdentifier + || prev_significant_token_type == TokenType::Number)) + return Token(TokenType::Dot, token_begin, ++pos); + + bool start_of_block = true; + ++pos; + while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(start_of_block, false, pos, end))) + { + ++pos; + start_of_block = false; + } + + /// exponentiation + if (pos + 1 < end && (*pos == 'e' || *pos == 'E')) + { + start_of_block = true; + ++pos; + + /// sign of exponent + if (pos + 1 < end && (*pos == '-' || *pos == '+')) + ++pos; + + while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(start_of_block, false, pos, end))) + { + ++pos; + start_of_block = false; + } + } + + return Token(TokenType::Number, token_begin, pos); + } + + case '+': + return Token(TokenType::Plus, token_begin, ++pos); + case '-': /// minus (-), arrow (->) or start of comment (--) + { + ++pos; + if (pos < end && *pos == '>') + return Token(TokenType::Arrow, token_begin, ++pos); + + if (pos < end && *pos == '-') + { + ++pos; + return comment_until_end_of_line(); + } + + return Token(TokenType::Minus, token_begin, pos); + } + case '*': + ++pos; + return Token(TokenType::Asterisk, token_begin, pos); + case '/': /// division (/) or start of comment (//, /*) + { + ++pos; + if (pos < end && (*pos == '/' || *pos == '*')) + { + if (*pos == '/') + { + ++pos; + return comment_until_end_of_line(); + } + + ++pos; + + /// Nested multiline comments are supported according to the SQL standard. + size_t nesting_level = 1; + + while (pos + 2 <= end) + { + if (pos[0] == '/' && pos[1] == '*') + { + pos += 2; + ++nesting_level; + } + else if (pos[0] == '*' && pos[1] == '/') + { + pos += 2; + --nesting_level; + + if (nesting_level == 0) + return Token(TokenType::Comment, token_begin, pos); + } + else + ++pos; + } + pos = end; + return Token(TokenType::ErrorMultilineCommentIsNotClosed, token_begin, pos); + } + return Token(TokenType::Slash, token_begin, pos); + } + case '#': /// start of single line comment, MySQL style + { /// PostgreSQL has some operators using '#' character. + /// For less ambiguity, we will recognize a comment only if # is followed by whitespace. + /// or #! as a special case for "shebang". + /// #hello - not a comment + /// # hello - a comment + /// #!/usr/bin/clickhouse-local --queries-file - a comment + ++pos; + if (pos < end && (*pos == ' ' || *pos == '!')) + return comment_until_end_of_line(); + return Token(TokenType::Error, token_begin, pos); + } + case '%': + return Token(TokenType::Percent, token_begin, ++pos); + case '=': /// =, == + { + ++pos; + if (pos < end && *pos == '=') + ++pos; + return Token(TokenType::Equals, token_begin, pos); + } + case '!': /// !=, !~, !~* + { + ++pos; + if (pos < end && *pos == '=') + return Token(TokenType::NotEquals, token_begin, ++pos); + if (pos < end && *pos == '~') + { + ++pos; + if (pos < end && *pos == '*') + return Token(TokenType::NotTildeAsterisk, token_begin, ++pos); + return Token(TokenType::NotTilde, token_begin, pos); + } + return Token(TokenType::ErrorSingleExclamationMark, token_begin, pos); + } + case '~': /// ~, ~* - regular expression match operators, PostgreSQL-style + { + ++pos; + if (pos < end && *pos == '*') + return Token(TokenType::TildeAsterisk, token_begin, ++pos); + return Token(TokenType::Tilde, token_begin, pos); + } + case '<': /// <, <=, <>, <=> + { + ++pos; + if (pos + 1 < end && *pos == '=' && *(pos + 1) == '>') + { + pos += 2; + return Token(TokenType::Spaceship, token_begin, pos); + } + if (pos < end && *pos == '=') + return Token(TokenType::LessOrEquals, token_begin, ++pos); + if (pos < end && *pos == '>') + return Token(TokenType::NotEquals, token_begin, ++pos); + return Token(TokenType::Less, token_begin, pos); + } + case '>': /// >, >= + { + ++pos; + if (pos < end && *pos == '=') + return Token(TokenType::GreaterOrEquals, token_begin, ++pos); + return Token(TokenType::Greater, token_begin, pos); + } + case '?': + return Token(TokenType::QuestionMark, token_begin, ++pos); + case '^': + return Token(TokenType::Caret, token_begin, ++pos); + case ':': + { + ++pos; + if (pos < end && *pos == ':') + return Token(TokenType::DoubleColon, token_begin, ++pos); + return Token(TokenType::Colon, token_begin, pos); + } + case '|': + { + ++pos; + if (pos < end && *pos == '|') + return Token(TokenType::Concatenation, token_begin, ++pos); + if (pos < end && *pos == '>') + return Token(TokenType::PipeOperator, token_begin, ++pos); + return Token(TokenType::PipeMark, token_begin, pos); + } + case '@': + { + ++pos; + if (pos < end && *pos == '@') + return Token(TokenType::DoubleAt, token_begin, ++pos); + return Token(TokenType::At, token_begin, pos); + } + case '\\': + { + ++pos; + if (pos < end && *pos == 'G') + return Token(TokenType::VerticalDelimiter, token_begin, ++pos); + return Token(TokenType::Error, token_begin, pos); + } + case '\xE2': + { + /// Mathematical minus symbol, UTF-8 + if (pos + 3 <= end && pos[1] == '\x88' && pos[2] == '\x92') + { + pos += 3; + return Token(TokenType::Minus, token_begin, pos); + } + /// Unicode quoted string, ‘Hello’ or “World”. + if (pos + 5 < end && pos[0] == '\xE2' && pos[1] == '\x80' && (pos[2] == '\x98' || pos[2] == '\x9C')) + { + const char expected_end_byte = pos[2] + 1; + TokenType success_token = pos[2] == '\x98' ? TokenType::StringLiteral : TokenType::QuotedIdentifier; + TokenType error_token = pos[2] == '\x98' ? TokenType::ErrorSingleQuoteIsNotClosed : TokenType::ErrorDoubleQuoteIsNotClosed; + pos += 3; + return quotedStringWithUnicodeQuotes(pos, token_begin, end, expected_end_byte, success_token, error_token); + } + /// Other characters starting at E2 can be parsed, see skipWhitespacesUTF8 + [[fallthrough]]; + } + default: + if (*pos == '$') + { + /// Try to capture a dollar sign as a start of heredoc + + const char * tag_end = find_first_symbols<'$'>(pos + 1, end); + if (tag_end != end) + { + size_t heredoc_size = tag_end + 1 - pos; + + bool is_valid_name = true; + for (const char * name_pos = pos + 1; name_pos < tag_end; ++name_pos) + { + if (!isWordCharASCII(*name_pos)) + { + is_valid_name = false; + break; + } + } + + if (is_valid_name) + { + size_t heredoc_end_position = std::string_view{tag_end + 1, end}.find(std::string_view{pos, heredoc_size}); + if (heredoc_end_position != std::string::npos) + { + pos = tag_end + 1 + heredoc_end_position + heredoc_size; + return Token(TokenType::HereDoc, token_begin, pos); + } + } + } + + if (((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end)) + { + /// Capture a standalone dollar sign + return Token(TokenType::DollarSign, token_begin, ++pos); + } + } + + if (pos + 2 < end && pos[1] == '\'' && (*pos == 'x' || *pos == 'b' || *pos == 'X' || *pos == 'B')) + { + return quotedHexOrBinString(pos, token_begin, end); + } + + if (isWordCharASCII(*pos) || *pos == '$') + { + ++pos; + while (pos < end && (isWordCharASCII(*pos) || *pos == '$')) + ++pos; + return Token(TokenType::BareWord, token_begin, pos); + } + + /// We will also skip unicode whitespaces in UTF-8 to support for queries copy-pasted from MS Word and similar. + pos = skipWhitespacesUTF8(pos, end); + if (pos > token_begin) + return Token(TokenType::Whitespace, token_begin, pos); + + ++pos; + while (pos < end && UTF8::isContinuationOctet(*pos)) + ++pos; + + return Token(TokenType::Error, token_begin, pos); + } +} + +#if !defined(LEXER_STANDALONE_BUILD) + +const char * getTokenName(TokenType type) +{ + switch (type) + { +#define M(TOKEN) \ + case TokenType::TOKEN: return #TOKEN; +APPLY_FOR_TOKENS(M) +#undef M + } +} + + +const char * getErrorTokenDescription(TokenType type) +{ + switch (type) + { + case TokenType::Error: + return "Unrecognized token"; + case TokenType::ErrorMultilineCommentIsNotClosed: + return "Multiline comment is not closed"; + case TokenType::ErrorSingleQuoteIsNotClosed: + return "Single quoted string is not closed"; + case TokenType::ErrorDoubleQuoteIsNotClosed: + return "Double quoted string is not closed"; + case TokenType::ErrorBackQuoteIsNotClosed: + return "Back quoted string is not closed"; + case TokenType::ErrorSingleExclamationMark: + return "Exclamation mark can only occur in !=, !~ and !~* operators"; + case TokenType::ErrorSinglePipeMark: + return "Pipe symbol could only occur in || operator"; + case TokenType::ErrorWrongNumber: + return "Wrong number"; + case TokenType::ErrorMaxQuerySizeExceeded: + return "Max query size exceeded (can be increased with the `max_query_size` setting)"; + default: + return "Not an error"; + } +} + +#else + +#include + +extern "C" +{ + +size_t clickhouse_lexer_size = sizeof(Lexer); + +void clickhouse_lexer_create(void * ptr, const char * begin, const char * end, size_t max_query_size) +{ + new(ptr) Lexer(begin, end, max_query_size); +} + +unsigned char clickhouse_lexer_next_token(void * ptr, const char ** out_token_begin, const char ** out_token_end) +{ + Token res = reinterpret_cast(ptr)->nextToken(); + *out_token_begin = res.begin; + *out_token_end = res.end; + return static_cast(res.type); +} + +int clickhouse_lexer_token_is_significant(unsigned char token) +{ + return token != static_cast(TokenType::Whitespace) && token != static_cast(TokenType::Comment); +} + +int clickhouse_lexer_token_is_error(unsigned char token) +{ + return token > static_cast(TokenType::EndOfStream); +} + +int clickhouse_lexer_token_is_end(unsigned char token) +{ + return token == static_cast(TokenType::EndOfStream); +} + +} + +#endif + +} diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h b/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h new file mode 100644 index 0000000000..cff1418bbd --- /dev/null +++ b/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h @@ -0,0 +1,150 @@ +// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/Lexer.h, +// which mirrors src/Parsers/Lexer.h of https://github.com/ClickHouse/ClickHouse at the +// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. + +#pragma once + +#if !defined(LEXER_STANDALONE_BUILD) + +#include +#include + +#endif + + +namespace DB +{ + +#define APPLY_FOR_TOKENS(M) \ + M(Whitespace) \ + M(Comment) \ + \ + M(BareWord) /** Either keyword (SELECT) or identifier (column) */ \ + \ + M(Number) /** Always non-negative. No leading plus. 123 or something like 123.456e12, 0x123p12 */ \ + M(StringLiteral) /** 'hello word', 'hello''word', 'hello\'word\\' */ \ + \ + M(QuotedIdentifier) /** "x", `x` */ \ + \ + M(OpeningRoundBracket) \ + M(ClosingRoundBracket) \ + \ + M(OpeningSquareBracket) \ + M(ClosingSquareBracket) \ + \ + M(OpeningCurlyBrace) \ + M(ClosingCurlyBrace) \ + \ + M(Comma) \ + M(Semicolon) \ + M(VerticalDelimiter) /** Vertical delimiter \G */ \ + M(Dot) /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \ + /** Need to be distinguished from floating point number with omitted integer part: .1 */ \ + \ + M(Asterisk) /** Could be used as multiplication operator or on it's own: "SELECT *" */ \ + \ + M(HereDoc) \ + \ + M(DollarSign) \ + M(Plus) \ + M(Minus) \ + M(Slash) \ + M(Percent) \ + M(Arrow) /** ->. Should be distinguished from minus operator. */ \ + M(QuestionMark) \ + M(Colon) \ + M(Caret) \ + M(DoubleColon) \ + M(Equals) \ + M(NotEquals) \ + M(Less) \ + M(Greater) \ + M(LessOrEquals) \ + M(GreaterOrEquals) \ + M(Spaceship) /** <=>. Used in MySQL for NULL-safe equality comparison. */ \ + M(PipeMark) \ + M(PipeOperator) /** |>. Pipe operator: FROM t |> WHERE x |> SELECT y */ \ + M(Concatenation) /** String concatenation operator: || */ \ + M(Tilde) /** ~. Used in PostgreSQL as the regular expression match operator. */ \ + M(TildeAsterisk) /** ~*. Used in PostgreSQL as the case-insensitive regular expression match operator. */ \ + M(NotTilde) /** !~. Used in PostgreSQL as the regular expression not-match operator. */ \ + M(NotTildeAsterisk) /** !~*. Used in PostgreSQL as the case-insensitive regular expression not-match operator. */ \ + \ + M(At) /** @. Used for specifying user names and also for MySQL-style variables. */ \ + M(DoubleAt) /** @@. Used for MySQL-style global variables. */ \ + \ + /** Order is important. EndOfStream goes after all usual tokens, and special error tokens goes after EndOfStream. */ \ + \ + M(EndOfStream) \ + \ + /** Something unrecognized. */ \ + M(Error) \ + /** Something is wrong and we have more information. */ \ + M(ErrorMultilineCommentIsNotClosed) \ + M(ErrorSingleQuoteIsNotClosed) \ + M(ErrorDoubleQuoteIsNotClosed) \ + M(ErrorBackQuoteIsNotClosed) \ + M(ErrorSingleExclamationMark) \ + M(ErrorSinglePipeMark) \ + M(ErrorWrongNumber) \ + M(ErrorMaxQuerySizeExceeded) \ + + +enum class TokenType : uint8_t +{ +#define M(TOKEN) TOKEN, +APPLY_FOR_TOKENS(M) +#undef M +}; + +#if !defined(LEXER_STANDALONE_BUILD) +const char * getTokenName(TokenType type); +const char * getErrorTokenDescription(TokenType type); +#endif + + +struct Token +{ + TokenType type; + const char * begin; + const char * end; + + size_t size() const { return end - begin; } + + Token() = default; + Token(TokenType type_, const char * begin_, const char * end_) : type(type_), begin(begin_), end(end_) {} + + bool isSignificant() const { return type != TokenType::Whitespace && type != TokenType::Comment; } + bool isError() const { return type > TokenType::EndOfStream; } + bool isEnd() const { return type == TokenType::EndOfStream; } +}; + + +class Lexer +{ +public: + Lexer(const char * begin_, const char * end_, size_t max_query_size_ = 0) + : begin(begin_), pos(begin_), end(end_), + max_query_size(max_query_size_ <= max_query_size_limit ? max_query_size_ : max_query_size_limit) + { + } + + Token nextToken(); + +private: + const char * const begin; + const char * pos; + const char * const end; + + const size_t max_query_size; + + /// Some reasonable size to at least avoid pointer overflows. + static constexpr size_t max_query_size_limit = 1'000'000'000; + + Token nextTokenImpl(); + + /// This is needed to disambiguate tuple access operator from floating point number (.1). + TokenType prev_significant_token_type = TokenType::Whitespace; /// No previous token. +}; + +} diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h b/fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h new file mode 100644 index 0000000000..f49871c5e6 --- /dev/null +++ b/fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h @@ -0,0 +1,180 @@ +// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/LexerStandalone.h, +// which mirrors src/Parsers/LexerStandalone.h of https://github.com/ClickHouse/ClickHouse at the +// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. + +#pragma once + +/// This allows building Lexer without any dependencies or includes for WebAssembly or Emscripten. + +#define chassert(...) + +#define size_t unsigned long +#define uint8_t unsigned char +#ifndef __cpp_char8_t + #define char8_t unsigned char +#endif + + +namespace // NOLINT +{ + +template +inline const char * find_first_symbols(const char * begin, const char * end) +{ + for (; begin != end; ++begin) + if (((*begin == symbols) || ...)) + break; + return begin; +} + +template +inline const char * find_first_not_symbols(const char * begin, const char * end) +{ + for (; begin != end; ++begin) + if (((*begin != symbols) && ...)) + break; + return begin; +} + +inline bool isNumericASCII(char c) +{ + return (c >= '0' && c <= '9'); +} + +inline bool isLowerAlphaASCII(char c) +{ + return (c >= 'a' && c <= 'z'); +} + +inline bool isUpperAlphaASCII(char c) +{ + return (c >= 'A' && c <= 'Z'); +} + +inline bool isAlphaASCII(char c) +{ + return isLowerAlphaASCII(c) || isUpperAlphaASCII(c); +} + +inline bool isAlphaNumericASCII(char c) +{ + return isAlphaASCII(c) + || isNumericASCII(c); +} + +inline bool isHexDigit(char c) +{ + return isNumericASCII(c) + || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F'); +} + +inline bool isNumberSeparator(bool is_start_of_block, bool is_hex, const char * pos, const char * end) +{ + if (*pos != '_') + return false; + if (is_start_of_block && *pos == '_') + return false; // e.g. _123, 12e_3 + if (pos + 1 < end && !(is_hex ? isHexDigit(pos[1]) : isNumericASCII(pos[1]))) + return false; // e.g. 1__2, 1_., 1_e, 1_p, 1_; + if (pos + 1 == end) + return false; // e.g. 12_ + return true; +} + +inline bool isWhitespaceASCII(char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +inline bool isWordCharASCII(char c) +{ + return isAlphaNumericASCII(c) + || c == '_'; +} + +namespace std +{ +namespace string +{ +constexpr size_t npos = -1; +} + +struct string_view +{ + const char * ptr; + size_t size; + + string_view(const char * ptr_, size_t size_) : ptr(ptr_), size(size_) + { + } + + string_view(const char * begin, const char * end) : ptr(begin), size(end - begin) + { + } + + inline size_t find(string_view other) const + { + for (size_t offset = 0; offset + other.size <= size; ++offset) + { + size_t other_pos = 0; + for (; other_pos < other.size; ++other_pos) + if (ptr[offset + other_pos] != other.ptr[other_pos]) + break; + if (other_pos == other.size) + return offset; + } + return string::npos; + } +}; +} + +inline const char * skipWhitespacesUTF8(const char * pos, const char * end) +{ + while (pos < end) + { + if (isWhitespaceASCII(*pos)) + { + ++pos; + } + else + { + const uint8_t * upos = reinterpret_cast(pos); + + if (pos + 1 < end && upos[0] == 0xC2 && (upos[1] == 0x85 || upos[1] == 0xA0)) + { + pos += 2; + } + else if (pos + 2 < end + && ((upos[0] == 0xE1 && upos[1] == 0xA0 && upos[2] == 0x8E) + || (upos[0] == 0xE2 + && ((upos[1] == 0x80 + && ((upos[2] >= 0x80 && upos[2] <= 0x8A) + || (upos[2] >= 0xA8 && upos[2] <= 0xA9) + || (upos[2] >= 0x8B && upos[2] <= 0x8D) + || (upos[2] == 0xAF))) + || (upos[1] == 0x81 && (upos[2] == 0x9F || upos[2] == 0xA0)))) + || (upos[0] == 0xE3 && upos[1] == 0x80 && upos[2] == 0x80) + || (upos[0] == 0xEF && upos[1] == 0xBB && upos[2] == 0xBF))) + { + pos += 3; + } + else + break; + } + } + + return pos; +} + +namespace UTF8 +{ +inline bool isContinuationOctet(char8_t octet) +{ + return (octet & 0b11000000u) == 0b10000000u; +} +} + +} + +inline void * operator new(size_t, void * p) noexcept { return p; } diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h b/fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h new file mode 100644 index 0000000000..5145884dc7 --- /dev/null +++ b/fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h @@ -0,0 +1,22 @@ +// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/clickhouse_lexer.h, +// which mirrors src/Parsers/clickhouse_lexer.h of https://github.com/ClickHouse/ClickHouse at the +// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +extern size_t clickhouse_lexer_size; +void clickhouse_lexer_create(void * ptr, const char * begin, const char * end, size_t max_query_size); +unsigned char clickhouse_lexer_next_token(void * ptr, const char ** out_token_begin, const char ** out_token_end); +int clickhouse_lexer_token_is_significant(unsigned char token); +int clickhouse_lexer_token_is_error(unsigned char token); +int clickhouse_lexer_token_is_end(unsigned char token); + +#ifdef __cplusplus +} +#endif diff --git a/fuzz/vendor/clickhouse-lexer/README.md b/fuzz/vendor/clickhouse-lexer/README.md new file mode 100644 index 0000000000..bb04a1be8f --- /dev/null +++ b/fuzz/vendor/clickhouse-lexer/README.md @@ -0,0 +1,14 @@ +# ClickHouse lexer + +`Parsers/` holds the lexer of ClickHouse, `src/Parsers/` in +, as redistributed by +. `COMMIT` names the ClickHouse +commit the files were taken from. They are © ClickHouse, Inc. and licensed under +the Apache License 2.0, see `LICENSE`. + +`fuzz_clickhouse_lexes` compiles them natively with `LEXER_STANDALONE_BUILD`, the +same configuration ClickHouse uses for its WebAssembly build, and calls the C ABI +declared in `Parsers/clickhouse_lexer.h`. + +To update, copy `upstream/` from clickhouse-lexer over `Parsers/` and `COMMIT`, and +re-add the provenance header at the top of each file. From 9cf91c073199d11428564829af9523add93caac0 Mon Sep 17 00:00:00 2001 From: s5dsn-eqee Date: Sat, 17 Oct 2026 22:12:19 +0200 Subject: [PATCH 2/2] Use the clickhouse-lexer-sys crate instead of vendoring the lexer --- docs/fuzzing.md | 2 +- fuzz/Cargo.toml | 6 +- fuzz/build.rs | 34 - fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs | 52 +- fuzz/vendor/clickhouse-lexer/COMMIT | 1 - fuzz/vendor/clickhouse-lexer/LICENSE | 203 ------ .../vendor/clickhouse-lexer/Parsers/Lexer.cpp | 650 ------------------ fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h | 150 ---- .../Parsers/LexerStandalone.h | 180 ----- .../Parsers/clickhouse_lexer.h | 22 - fuzz/vendor/clickhouse-lexer/README.md | 14 - 11 files changed, 10 insertions(+), 1304 deletions(-) delete mode 100644 fuzz/build.rs delete mode 100644 fuzz/vendor/clickhouse-lexer/COMMIT delete mode 100644 fuzz/vendor/clickhouse-lexer/LICENSE delete mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp delete mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h delete mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h delete mode 100644 fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h delete mode 100644 fuzz/vendor/clickhouse-lexer/README.md diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 861c4dc93b..cd5054903b 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -37,7 +37,7 @@ rendered statement no longer parses. `fuzz_duckdb_accepts` parses the input with DuckDB's own parser and fails when DuckDB accepts a `SELECT` that `DuckDbDialect` rejects. -`fuzz_clickhouse_lexes` tokenizes the input with ClickHouse's own lexer, vendored under `fuzz/vendor/clickhouse-lexer/`, and fails when it reaches the end without an error token but `ClickHouseDialect` fails to tokenize. ClickHouse has no standalone parser, so the check stops at the tokenizer. +`fuzz_clickhouse_lexes` tokenizes the input with ClickHouse's own lexer, from the [`clickhouse-lexer-sys`](https://crates.io/crates/clickhouse-lexer-sys) crate, and fails when it reaches the end without an error token but `ClickHouseDialect` fails to tokenize. ClickHouse has no standalone parser, so the check stops at the tokenizer; a lone `$`, which lexes but never parses, counts as rejected. The four oracle targets compile their engine from source, so each builds only with its feature, `sqlite`, `postgres`, `duckdb` or `clickhouse`, as in `cargo +nightly fuzz run --features sqlite fuzz_sqlite_accepts fuzz_seeds`. `cargo +nightly fuzz build --all-features` builds every target. diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index dabd6090bb..72a589b624 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -29,18 +29,16 @@ libfuzzer-sys = "0.4" libsqlite3-sys = { version = "0.38", features = ["bundled"], optional = true } pg_query = { version = "6.2", optional = true } libduckdb-sys = { version = "1.10505.0", features = ["bundled", "json"], optional = true } +clickhouse-lexer-sys = { version = "0.1", optional = true } serde_json = { version = "1", optional = true } sqlparser = { path = ".." } -[build-dependencies] -cc = { version = "1", optional = true } - # The oracle targets build their engines from source, so each compiles only when its feature is on. [features] sqlite = ["dep:libsqlite3-sys"] postgres = ["dep:pg_query"] duckdb = ["dep:libduckdb-sys", "dep:serde_json"] -clickhouse = ["dep:cc"] +clickhouse = ["dep:clickhouse-lexer-sys"] # Prevent this from interfering with workspaces [workspace] diff --git a/fuzz/build.rs b/fuzz/build.rs deleted file mode 100644 index 9b9feff18d..0000000000 --- a/fuzz/build.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -fn main() { - #[cfg(feature = "clickhouse")] - { - let dir = "vendor/clickhouse-lexer"; - println!("cargo:rerun-if-changed={dir}"); - cc::Build::new() - .cpp(true) - .std("c++20") - .include(dir) - .file(format!("{dir}/Parsers/Lexer.cpp")) - .define("LEXER_STANDALONE_BUILD", None) - .flag("-fno-exceptions") - .flag("-fno-rtti") - .warnings(false) - .compile("clickhouse_lexer"); - } -} diff --git a/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs b/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs index a810166c1c..44d68cc035 100644 --- a/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs +++ b/fuzz/fuzz_targets/fuzz_clickhouse_lexes.rs @@ -17,54 +17,17 @@ #![no_main] +use clickhouse_lexer_sys::{Lexer, TokenKind, UPSTREAM_COMMIT}; use libfuzzer_sys::fuzz_target; use sqlparser::dialect::ClickHouseDialect; use sqlparser::tokenizer::Tokenizer; -use std::ffi::{c_char, c_int, c_uchar, c_void}; -const COMMIT: &str = include_str!("../vendor/clickhouse-lexer/COMMIT"); - -// C ABI of the vendored lexer, see fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h. -extern "C" { - static clickhouse_lexer_size: usize; - fn clickhouse_lexer_create( - ptr: *mut c_void, - begin: *const c_char, - end: *const c_char, - max_query_size: usize, - ); - fn clickhouse_lexer_next_token( - ptr: *mut c_void, - out_token_begin: *mut *const c_char, - out_token_end: *mut *const c_char, - ) -> c_uchar; - fn clickhouse_lexer_token_is_error(token: c_uchar) -> c_int; - fn clickhouse_lexer_token_is_end(token: c_uchar) -> c_int; -} - -/// Whether ClickHouse's lexer reads `sql` to the end without an error token. +/// Whether ClickHouse's lexer reads `sql` to the end without an error token or a lone `$`. +/// +/// A lone `$` lexes as `DollarSign`, which only the JSONPath parser consumes, so no +/// statement holding one parses. fn clickhouse_lexes(sql: &str) -> bool { - // SAFETY: the symbol is a plain `size_t` the C++ side initializes statically. - let size = unsafe { clickhouse_lexer_size }; - // u64 slots keep the pointer fields of the C++ Lexer aligned. - let mut lexer = vec![0u64; size.div_ceil(8)]; - let ptr = lexer.as_mut_ptr().cast::(); - let begin = sql.as_ptr().cast::(); - // SAFETY: `lexer` holds `clickhouse_lexer_size` bytes and `sql` outlives every call below. - unsafe { - clickhouse_lexer_create(ptr, begin, begin.add(sql.len()), 0); - loop { - let mut token_begin = std::ptr::null(); - let mut token_end = std::ptr::null(); - let token = clickhouse_lexer_next_token(ptr, &mut token_begin, &mut token_end); - if clickhouse_lexer_token_is_error(token) != 0 { - return false; - } - if clickhouse_lexer_token_is_end(token) != 0 { - return true; - } - } - } + Lexer::new(sql).all(|token| !token.kind.is_error() && token.kind != TokenKind::DollarSign) } fuzz_target!(|sql: &str| { @@ -73,8 +36,7 @@ fuzz_target!(|sql: &str| { } if let Err(err) = Tokenizer::new(&ClickHouseDialect {}, sql).tokenize() { panic!( - "ClickHouse lexer at {} accepts SQL the ClickHouse dialect tokenizer rejects\n sql: {sql:?}\n error: {err}", - COMMIT.trim() + "ClickHouse lexer at {UPSTREAM_COMMIT} accepts SQL the ClickHouse dialect tokenizer rejects\n sql: {sql:?}\n error: {err}" ) } }); diff --git a/fuzz/vendor/clickhouse-lexer/COMMIT b/fuzz/vendor/clickhouse-lexer/COMMIT deleted file mode 100644 index 70ee81fc65..0000000000 --- a/fuzz/vendor/clickhouse-lexer/COMMIT +++ /dev/null @@ -1 +0,0 @@ -55c2563639c7ab7d90aa85f52351ec35c936bae7 diff --git a/fuzz/vendor/clickhouse-lexer/LICENSE b/fuzz/vendor/clickhouse-lexer/LICENSE deleted file mode 100644 index f1da7e1a39..0000000000 --- a/fuzz/vendor/clickhouse-lexer/LICENSE +++ /dev/null @@ -1,203 +0,0 @@ -Copyright 2016-2026 ClickHouse, Inc. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2016-2026 ClickHouse, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp b/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp deleted file mode 100644 index e2f3a41374..0000000000 --- a/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.cpp +++ /dev/null @@ -1,650 +0,0 @@ -// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/Lexer.cpp, -// which mirrors src/Parsers/Lexer.cpp of https://github.com/ClickHouse/ClickHouse at the -// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. - -#if !defined(LEXER_STANDALONE_BUILD) - -#include -#include -#include -#include -#include - -#else /// This allows building Lexer without any dependencies or includes for WebAssembly or Emscripten. - -#include -#include - -#endif - - -namespace DB -{ - -namespace -{ - -/// This must be consistent with functions in ReadHelpers.h -template -Token quotedString(const char *& pos, const char * const token_begin, const char * const end, - TokenType success_token, TokenType error_token) -{ - ++pos; - while (true) - { - pos = find_first_symbols(pos, end); - if (pos >= end) - return Token(error_token, token_begin, end); - - if (*pos == quote) - { - ++pos; - if (pos < end && *pos == quote) - { - ++pos; - continue; - } - return Token(success_token, token_begin, pos); - } - - if (*pos == '\\') - { - ++pos; - if (pos >= end) - return Token(error_token, token_begin, end); - ++pos; - continue; - } - - chassert(false); - } -} - -Token quotedStringWithUnicodeQuotes(const char *& pos, const char * const token_begin, const char * const end, - char expected_end_byte, TokenType success_token, TokenType error_token) -{ - /// ‘: e2 80 98 - /// ’: e2 80 99 - /// “: e2 80 9c - /// ”: e2 80 9d - - while (true) - { - pos = find_first_symbols<'\xE2'>(pos, end); - if (pos + 2 >= end) - return Token(error_token, token_begin, end); - - if (pos[0] == '\xE2' && pos[1] == '\x80' && pos[2] == expected_end_byte) - { - pos += 3; - return Token(success_token, token_begin, pos); - } - - ++pos; - } -} - -Token quotedHexOrBinString(const char *& pos, const char * const token_begin, const char * const end) -{ - constexpr char quote = '\''; - - chassert(pos[1] == quote); - - bool hex = (*pos == 'x' || *pos == 'X'); - - pos += 2; - - if (hex) - { - while (pos < end && isHexDigit(*pos)) - ++pos; - } - else - { - pos = find_first_not_symbols<'0', '1'>(pos, end); - } - - if (pos >= end || *pos != quote) - { - pos = end; - return Token(TokenType::ErrorSingleQuoteIsNotClosed, token_begin, end); - } - - ++pos; - return Token(TokenType::StringLiteral, token_begin, pos); -} - -} - - -Token Lexer::nextToken() -{ - Token res = nextTokenImpl(); - if (begin && max_query_size && res.end > begin + max_query_size) - res.type = TokenType::ErrorMaxQuerySizeExceeded; - if (res.isSignificant()) - prev_significant_token_type = res.type; - return res; -} - - -Token Lexer::nextTokenImpl() -{ - if (pos >= end) - return Token(TokenType::EndOfStream, end, end); - - const char * const token_begin = pos; - - auto comment_until_end_of_line = [&]() mutable - { - pos = find_first_symbols<'\n'>(pos, end); /// This means that newline in single-line comment cannot be escaped. - return Token(TokenType::Comment, token_begin, pos); - }; - - switch (*pos) - { - case ' ': - case '\t': - case '\n': - case '\r': - case '\f': - case '\v': - { - ++pos; - while (pos < end && isWhitespaceASCII(*pos)) - ++pos; - return Token(TokenType::Whitespace, token_begin, pos); - } - - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - { - /// The task is not to parse a number or check correctness, but only to skip it. - - /// Disambiguation: if previous token was dot, then we could parse only simple integer, - /// for chained tuple access operators (x.1.1) to work. - // Otherwise it will be tokenized as x . 1.1, not as x . 1 . 1 - if (prev_significant_token_type == TokenType::Dot) - { - ++pos; - while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(false, false, pos, end))) - ++pos; - } - else - { - bool start_of_block = false; - /// 0x, 0b - bool hex = false; - if (pos + 2 < end && *pos == '0' && (pos[1] == 'x' || pos[1] == 'b' || pos[1] == 'X' || pos[1] == 'B')) - { - bool is_valid = false; - if (pos[1] == 'x' || pos[1] == 'X') - { - if (isHexDigit(pos[2])) - { - hex = true; - is_valid = true; // hex - } - } - else if (pos[2] == '0' || pos[2] == '1') - is_valid = true; // bin - if (is_valid) - { - pos += 2; - start_of_block = true; - } - else - ++pos; // consume the leading zero - could be an identifier - } - else - ++pos; - - while (pos < end && ((hex ? isHexDigit(*pos) : isNumericASCII(*pos)) || isNumberSeparator(start_of_block, hex, pos, end))) - { - ++pos; - start_of_block = false; - } - - /// decimal point - if (pos < end && *pos == '.') - { - start_of_block = true; - ++pos; - while (pos < end && ((hex ? isHexDigit(*pos) : isNumericASCII(*pos)) || isNumberSeparator(start_of_block, hex, pos, end))) - { - ++pos; - start_of_block = false; - } - } - - /// exponentiation (base 10 or base 2) - if (pos + 1 < end && (hex ? (*pos == 'p' || *pos == 'P') : (*pos == 'e' || *pos == 'E'))) - { - start_of_block = true; - ++pos; - - /// sign of exponent. It is always decimal. - if (pos + 1 < end && (*pos == '-' || *pos == '+')) - ++pos; - - while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(start_of_block, false, pos, end))) - { - ++pos; - start_of_block = false; - } - } - } - - /// Try to parse it to a identifier(1identifier_name), otherwise it return ErrorWrongNumber - if (pos < end && isWordCharASCII(*pos)) - { - ++pos; - while (pos < end && isWordCharASCII(*pos)) - ++pos; - - for (const char * iterator = token_begin; iterator < pos; ++iterator) - { - if (!isWordCharASCII(*iterator) && *iterator != '$') - return Token(TokenType::ErrorWrongNumber, token_begin, pos); - } - - return Token(TokenType::BareWord, token_begin, pos); - } - - return Token(TokenType::Number, token_begin, pos); - } - - case '\'': - return quotedString<'\''>(pos, token_begin, end, TokenType::StringLiteral, TokenType::ErrorSingleQuoteIsNotClosed); - case '"': - return quotedString<'"'>(pos, token_begin, end, TokenType::QuotedIdentifier, TokenType::ErrorDoubleQuoteIsNotClosed); - case '`': - return quotedString<'`'>(pos, token_begin, end, TokenType::QuotedIdentifier, TokenType::ErrorBackQuoteIsNotClosed); - - case '(': - return Token(TokenType::OpeningRoundBracket, token_begin, ++pos); - case ')': - return Token(TokenType::ClosingRoundBracket, token_begin, ++pos); - case '[': - return Token(TokenType::OpeningSquareBracket, token_begin, ++pos); - case ']': - return Token(TokenType::ClosingSquareBracket, token_begin, ++pos); - case '{': - return Token(TokenType::OpeningCurlyBrace, token_begin, ++pos); - case '}': - return Token(TokenType::ClosingCurlyBrace, token_begin, ++pos); - case ',': - return Token(TokenType::Comma, token_begin, ++pos); - case ';': - return Token(TokenType::Semicolon, token_begin, ++pos); - - case '.': /// qualifier, tuple access operator or start of floating point number - { - /// Just after identifier or complex expression or number (for chained tuple access like x.1.1 to work properly). - if (pos > begin - && (!(pos + 1 < end && isNumericASCII(pos[1])) - || prev_significant_token_type == TokenType::ClosingRoundBracket - || prev_significant_token_type == TokenType::ClosingSquareBracket - || prev_significant_token_type == TokenType::BareWord - || prev_significant_token_type == TokenType::QuotedIdentifier - || prev_significant_token_type == TokenType::Number)) - return Token(TokenType::Dot, token_begin, ++pos); - - bool start_of_block = true; - ++pos; - while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(start_of_block, false, pos, end))) - { - ++pos; - start_of_block = false; - } - - /// exponentiation - if (pos + 1 < end && (*pos == 'e' || *pos == 'E')) - { - start_of_block = true; - ++pos; - - /// sign of exponent - if (pos + 1 < end && (*pos == '-' || *pos == '+')) - ++pos; - - while (pos < end && (isNumericASCII(*pos) || isNumberSeparator(start_of_block, false, pos, end))) - { - ++pos; - start_of_block = false; - } - } - - return Token(TokenType::Number, token_begin, pos); - } - - case '+': - return Token(TokenType::Plus, token_begin, ++pos); - case '-': /// minus (-), arrow (->) or start of comment (--) - { - ++pos; - if (pos < end && *pos == '>') - return Token(TokenType::Arrow, token_begin, ++pos); - - if (pos < end && *pos == '-') - { - ++pos; - return comment_until_end_of_line(); - } - - return Token(TokenType::Minus, token_begin, pos); - } - case '*': - ++pos; - return Token(TokenType::Asterisk, token_begin, pos); - case '/': /// division (/) or start of comment (//, /*) - { - ++pos; - if (pos < end && (*pos == '/' || *pos == '*')) - { - if (*pos == '/') - { - ++pos; - return comment_until_end_of_line(); - } - - ++pos; - - /// Nested multiline comments are supported according to the SQL standard. - size_t nesting_level = 1; - - while (pos + 2 <= end) - { - if (pos[0] == '/' && pos[1] == '*') - { - pos += 2; - ++nesting_level; - } - else if (pos[0] == '*' && pos[1] == '/') - { - pos += 2; - --nesting_level; - - if (nesting_level == 0) - return Token(TokenType::Comment, token_begin, pos); - } - else - ++pos; - } - pos = end; - return Token(TokenType::ErrorMultilineCommentIsNotClosed, token_begin, pos); - } - return Token(TokenType::Slash, token_begin, pos); - } - case '#': /// start of single line comment, MySQL style - { /// PostgreSQL has some operators using '#' character. - /// For less ambiguity, we will recognize a comment only if # is followed by whitespace. - /// or #! as a special case for "shebang". - /// #hello - not a comment - /// # hello - a comment - /// #!/usr/bin/clickhouse-local --queries-file - a comment - ++pos; - if (pos < end && (*pos == ' ' || *pos == '!')) - return comment_until_end_of_line(); - return Token(TokenType::Error, token_begin, pos); - } - case '%': - return Token(TokenType::Percent, token_begin, ++pos); - case '=': /// =, == - { - ++pos; - if (pos < end && *pos == '=') - ++pos; - return Token(TokenType::Equals, token_begin, pos); - } - case '!': /// !=, !~, !~* - { - ++pos; - if (pos < end && *pos == '=') - return Token(TokenType::NotEquals, token_begin, ++pos); - if (pos < end && *pos == '~') - { - ++pos; - if (pos < end && *pos == '*') - return Token(TokenType::NotTildeAsterisk, token_begin, ++pos); - return Token(TokenType::NotTilde, token_begin, pos); - } - return Token(TokenType::ErrorSingleExclamationMark, token_begin, pos); - } - case '~': /// ~, ~* - regular expression match operators, PostgreSQL-style - { - ++pos; - if (pos < end && *pos == '*') - return Token(TokenType::TildeAsterisk, token_begin, ++pos); - return Token(TokenType::Tilde, token_begin, pos); - } - case '<': /// <, <=, <>, <=> - { - ++pos; - if (pos + 1 < end && *pos == '=' && *(pos + 1) == '>') - { - pos += 2; - return Token(TokenType::Spaceship, token_begin, pos); - } - if (pos < end && *pos == '=') - return Token(TokenType::LessOrEquals, token_begin, ++pos); - if (pos < end && *pos == '>') - return Token(TokenType::NotEquals, token_begin, ++pos); - return Token(TokenType::Less, token_begin, pos); - } - case '>': /// >, >= - { - ++pos; - if (pos < end && *pos == '=') - return Token(TokenType::GreaterOrEquals, token_begin, ++pos); - return Token(TokenType::Greater, token_begin, pos); - } - case '?': - return Token(TokenType::QuestionMark, token_begin, ++pos); - case '^': - return Token(TokenType::Caret, token_begin, ++pos); - case ':': - { - ++pos; - if (pos < end && *pos == ':') - return Token(TokenType::DoubleColon, token_begin, ++pos); - return Token(TokenType::Colon, token_begin, pos); - } - case '|': - { - ++pos; - if (pos < end && *pos == '|') - return Token(TokenType::Concatenation, token_begin, ++pos); - if (pos < end && *pos == '>') - return Token(TokenType::PipeOperator, token_begin, ++pos); - return Token(TokenType::PipeMark, token_begin, pos); - } - case '@': - { - ++pos; - if (pos < end && *pos == '@') - return Token(TokenType::DoubleAt, token_begin, ++pos); - return Token(TokenType::At, token_begin, pos); - } - case '\\': - { - ++pos; - if (pos < end && *pos == 'G') - return Token(TokenType::VerticalDelimiter, token_begin, ++pos); - return Token(TokenType::Error, token_begin, pos); - } - case '\xE2': - { - /// Mathematical minus symbol, UTF-8 - if (pos + 3 <= end && pos[1] == '\x88' && pos[2] == '\x92') - { - pos += 3; - return Token(TokenType::Minus, token_begin, pos); - } - /// Unicode quoted string, ‘Hello’ or “World”. - if (pos + 5 < end && pos[0] == '\xE2' && pos[1] == '\x80' && (pos[2] == '\x98' || pos[2] == '\x9C')) - { - const char expected_end_byte = pos[2] + 1; - TokenType success_token = pos[2] == '\x98' ? TokenType::StringLiteral : TokenType::QuotedIdentifier; - TokenType error_token = pos[2] == '\x98' ? TokenType::ErrorSingleQuoteIsNotClosed : TokenType::ErrorDoubleQuoteIsNotClosed; - pos += 3; - return quotedStringWithUnicodeQuotes(pos, token_begin, end, expected_end_byte, success_token, error_token); - } - /// Other characters starting at E2 can be parsed, see skipWhitespacesUTF8 - [[fallthrough]]; - } - default: - if (*pos == '$') - { - /// Try to capture a dollar sign as a start of heredoc - - const char * tag_end = find_first_symbols<'$'>(pos + 1, end); - if (tag_end != end) - { - size_t heredoc_size = tag_end + 1 - pos; - - bool is_valid_name = true; - for (const char * name_pos = pos + 1; name_pos < tag_end; ++name_pos) - { - if (!isWordCharASCII(*name_pos)) - { - is_valid_name = false; - break; - } - } - - if (is_valid_name) - { - size_t heredoc_end_position = std::string_view{tag_end + 1, end}.find(std::string_view{pos, heredoc_size}); - if (heredoc_end_position != std::string::npos) - { - pos = tag_end + 1 + heredoc_end_position + heredoc_size; - return Token(TokenType::HereDoc, token_begin, pos); - } - } - } - - if (((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end)) - { - /// Capture a standalone dollar sign - return Token(TokenType::DollarSign, token_begin, ++pos); - } - } - - if (pos + 2 < end && pos[1] == '\'' && (*pos == 'x' || *pos == 'b' || *pos == 'X' || *pos == 'B')) - { - return quotedHexOrBinString(pos, token_begin, end); - } - - if (isWordCharASCII(*pos) || *pos == '$') - { - ++pos; - while (pos < end && (isWordCharASCII(*pos) || *pos == '$')) - ++pos; - return Token(TokenType::BareWord, token_begin, pos); - } - - /// We will also skip unicode whitespaces in UTF-8 to support for queries copy-pasted from MS Word and similar. - pos = skipWhitespacesUTF8(pos, end); - if (pos > token_begin) - return Token(TokenType::Whitespace, token_begin, pos); - - ++pos; - while (pos < end && UTF8::isContinuationOctet(*pos)) - ++pos; - - return Token(TokenType::Error, token_begin, pos); - } -} - -#if !defined(LEXER_STANDALONE_BUILD) - -const char * getTokenName(TokenType type) -{ - switch (type) - { -#define M(TOKEN) \ - case TokenType::TOKEN: return #TOKEN; -APPLY_FOR_TOKENS(M) -#undef M - } -} - - -const char * getErrorTokenDescription(TokenType type) -{ - switch (type) - { - case TokenType::Error: - return "Unrecognized token"; - case TokenType::ErrorMultilineCommentIsNotClosed: - return "Multiline comment is not closed"; - case TokenType::ErrorSingleQuoteIsNotClosed: - return "Single quoted string is not closed"; - case TokenType::ErrorDoubleQuoteIsNotClosed: - return "Double quoted string is not closed"; - case TokenType::ErrorBackQuoteIsNotClosed: - return "Back quoted string is not closed"; - case TokenType::ErrorSingleExclamationMark: - return "Exclamation mark can only occur in !=, !~ and !~* operators"; - case TokenType::ErrorSinglePipeMark: - return "Pipe symbol could only occur in || operator"; - case TokenType::ErrorWrongNumber: - return "Wrong number"; - case TokenType::ErrorMaxQuerySizeExceeded: - return "Max query size exceeded (can be increased with the `max_query_size` setting)"; - default: - return "Not an error"; - } -} - -#else - -#include - -extern "C" -{ - -size_t clickhouse_lexer_size = sizeof(Lexer); - -void clickhouse_lexer_create(void * ptr, const char * begin, const char * end, size_t max_query_size) -{ - new(ptr) Lexer(begin, end, max_query_size); -} - -unsigned char clickhouse_lexer_next_token(void * ptr, const char ** out_token_begin, const char ** out_token_end) -{ - Token res = reinterpret_cast(ptr)->nextToken(); - *out_token_begin = res.begin; - *out_token_end = res.end; - return static_cast(res.type); -} - -int clickhouse_lexer_token_is_significant(unsigned char token) -{ - return token != static_cast(TokenType::Whitespace) && token != static_cast(TokenType::Comment); -} - -int clickhouse_lexer_token_is_error(unsigned char token) -{ - return token > static_cast(TokenType::EndOfStream); -} - -int clickhouse_lexer_token_is_end(unsigned char token) -{ - return token == static_cast(TokenType::EndOfStream); -} - -} - -#endif - -} diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h b/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h deleted file mode 100644 index cff1418bbd..0000000000 --- a/fuzz/vendor/clickhouse-lexer/Parsers/Lexer.h +++ /dev/null @@ -1,150 +0,0 @@ -// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/Lexer.h, -// which mirrors src/Parsers/Lexer.h of https://github.com/ClickHouse/ClickHouse at the -// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. - -#pragma once - -#if !defined(LEXER_STANDALONE_BUILD) - -#include -#include - -#endif - - -namespace DB -{ - -#define APPLY_FOR_TOKENS(M) \ - M(Whitespace) \ - M(Comment) \ - \ - M(BareWord) /** Either keyword (SELECT) or identifier (column) */ \ - \ - M(Number) /** Always non-negative. No leading plus. 123 or something like 123.456e12, 0x123p12 */ \ - M(StringLiteral) /** 'hello word', 'hello''word', 'hello\'word\\' */ \ - \ - M(QuotedIdentifier) /** "x", `x` */ \ - \ - M(OpeningRoundBracket) \ - M(ClosingRoundBracket) \ - \ - M(OpeningSquareBracket) \ - M(ClosingSquareBracket) \ - \ - M(OpeningCurlyBrace) \ - M(ClosingCurlyBrace) \ - \ - M(Comma) \ - M(Semicolon) \ - M(VerticalDelimiter) /** Vertical delimiter \G */ \ - M(Dot) /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \ - /** Need to be distinguished from floating point number with omitted integer part: .1 */ \ - \ - M(Asterisk) /** Could be used as multiplication operator or on it's own: "SELECT *" */ \ - \ - M(HereDoc) \ - \ - M(DollarSign) \ - M(Plus) \ - M(Minus) \ - M(Slash) \ - M(Percent) \ - M(Arrow) /** ->. Should be distinguished from minus operator. */ \ - M(QuestionMark) \ - M(Colon) \ - M(Caret) \ - M(DoubleColon) \ - M(Equals) \ - M(NotEquals) \ - M(Less) \ - M(Greater) \ - M(LessOrEquals) \ - M(GreaterOrEquals) \ - M(Spaceship) /** <=>. Used in MySQL for NULL-safe equality comparison. */ \ - M(PipeMark) \ - M(PipeOperator) /** |>. Pipe operator: FROM t |> WHERE x |> SELECT y */ \ - M(Concatenation) /** String concatenation operator: || */ \ - M(Tilde) /** ~. Used in PostgreSQL as the regular expression match operator. */ \ - M(TildeAsterisk) /** ~*. Used in PostgreSQL as the case-insensitive regular expression match operator. */ \ - M(NotTilde) /** !~. Used in PostgreSQL as the regular expression not-match operator. */ \ - M(NotTildeAsterisk) /** !~*. Used in PostgreSQL as the case-insensitive regular expression not-match operator. */ \ - \ - M(At) /** @. Used for specifying user names and also for MySQL-style variables. */ \ - M(DoubleAt) /** @@. Used for MySQL-style global variables. */ \ - \ - /** Order is important. EndOfStream goes after all usual tokens, and special error tokens goes after EndOfStream. */ \ - \ - M(EndOfStream) \ - \ - /** Something unrecognized. */ \ - M(Error) \ - /** Something is wrong and we have more information. */ \ - M(ErrorMultilineCommentIsNotClosed) \ - M(ErrorSingleQuoteIsNotClosed) \ - M(ErrorDoubleQuoteIsNotClosed) \ - M(ErrorBackQuoteIsNotClosed) \ - M(ErrorSingleExclamationMark) \ - M(ErrorSinglePipeMark) \ - M(ErrorWrongNumber) \ - M(ErrorMaxQuerySizeExceeded) \ - - -enum class TokenType : uint8_t -{ -#define M(TOKEN) TOKEN, -APPLY_FOR_TOKENS(M) -#undef M -}; - -#if !defined(LEXER_STANDALONE_BUILD) -const char * getTokenName(TokenType type); -const char * getErrorTokenDescription(TokenType type); -#endif - - -struct Token -{ - TokenType type; - const char * begin; - const char * end; - - size_t size() const { return end - begin; } - - Token() = default; - Token(TokenType type_, const char * begin_, const char * end_) : type(type_), begin(begin_), end(end_) {} - - bool isSignificant() const { return type != TokenType::Whitespace && type != TokenType::Comment; } - bool isError() const { return type > TokenType::EndOfStream; } - bool isEnd() const { return type == TokenType::EndOfStream; } -}; - - -class Lexer -{ -public: - Lexer(const char * begin_, const char * end_, size_t max_query_size_ = 0) - : begin(begin_), pos(begin_), end(end_), - max_query_size(max_query_size_ <= max_query_size_limit ? max_query_size_ : max_query_size_limit) - { - } - - Token nextToken(); - -private: - const char * const begin; - const char * pos; - const char * const end; - - const size_t max_query_size; - - /// Some reasonable size to at least avoid pointer overflows. - static constexpr size_t max_query_size_limit = 1'000'000'000; - - Token nextTokenImpl(); - - /// This is needed to disambiguate tuple access operator from floating point number (.1). - TokenType prev_significant_token_type = TokenType::Whitespace; /// No previous token. -}; - -} diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h b/fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h deleted file mode 100644 index f49871c5e6..0000000000 --- a/fuzz/vendor/clickhouse-lexer/Parsers/LexerStandalone.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/LexerStandalone.h, -// which mirrors src/Parsers/LexerStandalone.h of https://github.com/ClickHouse/ClickHouse at the -// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. - -#pragma once - -/// This allows building Lexer without any dependencies or includes for WebAssembly or Emscripten. - -#define chassert(...) - -#define size_t unsigned long -#define uint8_t unsigned char -#ifndef __cpp_char8_t - #define char8_t unsigned char -#endif - - -namespace // NOLINT -{ - -template -inline const char * find_first_symbols(const char * begin, const char * end) -{ - for (; begin != end; ++begin) - if (((*begin == symbols) || ...)) - break; - return begin; -} - -template -inline const char * find_first_not_symbols(const char * begin, const char * end) -{ - for (; begin != end; ++begin) - if (((*begin != symbols) && ...)) - break; - return begin; -} - -inline bool isNumericASCII(char c) -{ - return (c >= '0' && c <= '9'); -} - -inline bool isLowerAlphaASCII(char c) -{ - return (c >= 'a' && c <= 'z'); -} - -inline bool isUpperAlphaASCII(char c) -{ - return (c >= 'A' && c <= 'Z'); -} - -inline bool isAlphaASCII(char c) -{ - return isLowerAlphaASCII(c) || isUpperAlphaASCII(c); -} - -inline bool isAlphaNumericASCII(char c) -{ - return isAlphaASCII(c) - || isNumericASCII(c); -} - -inline bool isHexDigit(char c) -{ - return isNumericASCII(c) - || (c >= 'a' && c <= 'f') - || (c >= 'A' && c <= 'F'); -} - -inline bool isNumberSeparator(bool is_start_of_block, bool is_hex, const char * pos, const char * end) -{ - if (*pos != '_') - return false; - if (is_start_of_block && *pos == '_') - return false; // e.g. _123, 12e_3 - if (pos + 1 < end && !(is_hex ? isHexDigit(pos[1]) : isNumericASCII(pos[1]))) - return false; // e.g. 1__2, 1_., 1_e, 1_p, 1_; - if (pos + 1 == end) - return false; // e.g. 12_ - return true; -} - -inline bool isWhitespaceASCII(char c) -{ - return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; -} - -inline bool isWordCharASCII(char c) -{ - return isAlphaNumericASCII(c) - || c == '_'; -} - -namespace std -{ -namespace string -{ -constexpr size_t npos = -1; -} - -struct string_view -{ - const char * ptr; - size_t size; - - string_view(const char * ptr_, size_t size_) : ptr(ptr_), size(size_) - { - } - - string_view(const char * begin, const char * end) : ptr(begin), size(end - begin) - { - } - - inline size_t find(string_view other) const - { - for (size_t offset = 0; offset + other.size <= size; ++offset) - { - size_t other_pos = 0; - for (; other_pos < other.size; ++other_pos) - if (ptr[offset + other_pos] != other.ptr[other_pos]) - break; - if (other_pos == other.size) - return offset; - } - return string::npos; - } -}; -} - -inline const char * skipWhitespacesUTF8(const char * pos, const char * end) -{ - while (pos < end) - { - if (isWhitespaceASCII(*pos)) - { - ++pos; - } - else - { - const uint8_t * upos = reinterpret_cast(pos); - - if (pos + 1 < end && upos[0] == 0xC2 && (upos[1] == 0x85 || upos[1] == 0xA0)) - { - pos += 2; - } - else if (pos + 2 < end - && ((upos[0] == 0xE1 && upos[1] == 0xA0 && upos[2] == 0x8E) - || (upos[0] == 0xE2 - && ((upos[1] == 0x80 - && ((upos[2] >= 0x80 && upos[2] <= 0x8A) - || (upos[2] >= 0xA8 && upos[2] <= 0xA9) - || (upos[2] >= 0x8B && upos[2] <= 0x8D) - || (upos[2] == 0xAF))) - || (upos[1] == 0x81 && (upos[2] == 0x9F || upos[2] == 0xA0)))) - || (upos[0] == 0xE3 && upos[1] == 0x80 && upos[2] == 0x80) - || (upos[0] == 0xEF && upos[1] == 0xBB && upos[2] == 0xBF))) - { - pos += 3; - } - else - break; - } - } - - return pos; -} - -namespace UTF8 -{ -inline bool isContinuationOctet(char8_t octet) -{ - return (octet & 0b11000000u) == 0b10000000u; -} -} - -} - -inline void * operator new(size_t, void * p) noexcept { return p; } diff --git a/fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h b/fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h deleted file mode 100644 index 5145884dc7..0000000000 --- a/fuzz/vendor/clickhouse-lexer/Parsers/clickhouse_lexer.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copied, with only this header added, from https://github.com/ClickHouse/clickhouse-lexer, upstream/clickhouse_lexer.h, -// which mirrors src/Parsers/clickhouse_lexer.h of https://github.com/ClickHouse/ClickHouse at the -// commit named in ../COMMIT. Copyright ClickHouse, Inc., Apache License 2.0, see ../LICENSE. - -#pragma once - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -extern size_t clickhouse_lexer_size; -void clickhouse_lexer_create(void * ptr, const char * begin, const char * end, size_t max_query_size); -unsigned char clickhouse_lexer_next_token(void * ptr, const char ** out_token_begin, const char ** out_token_end); -int clickhouse_lexer_token_is_significant(unsigned char token); -int clickhouse_lexer_token_is_error(unsigned char token); -int clickhouse_lexer_token_is_end(unsigned char token); - -#ifdef __cplusplus -} -#endif diff --git a/fuzz/vendor/clickhouse-lexer/README.md b/fuzz/vendor/clickhouse-lexer/README.md deleted file mode 100644 index bb04a1be8f..0000000000 --- a/fuzz/vendor/clickhouse-lexer/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# ClickHouse lexer - -`Parsers/` holds the lexer of ClickHouse, `src/Parsers/` in -, as redistributed by -. `COMMIT` names the ClickHouse -commit the files were taken from. They are © ClickHouse, Inc. and licensed under -the Apache License 2.0, see `LICENSE`. - -`fuzz_clickhouse_lexes` compiles them natively with `LEXER_STANDALONE_BUILD`, the -same configuration ClickHouse uses for its WebAssembly build, and calls the C ABI -declared in `Parsers/clickhouse_lexer.h`. - -To update, copy `upstream/` from clickhouse-lexer over `Parsers/` and `COMMIT`, and -re-add the provenance header at the top of each file.