Skip to content

Commit b50f776

Browse files
Merge main into PR apache#2193
2 parents b4c44c0 + c70b5b1 commit b50f776

59 files changed

Lines changed: 3765 additions & 725 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.asf.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ notifications:
2424
commits: commits@datafusion.apache.org
2525
issues: github@datafusion.apache.org
2626
pullrequests: github@datafusion.apache.org
27+
discussions: github@datafusion.apache.org
2728
github:
2829
description: "Extensible SQL Lexer and Parser for Rust"
2930
labels:
@@ -36,6 +37,7 @@ github:
3637
rebase: false
3738
features:
3839
issues: true
40+
discussions: true
3941
protected_branches:
4042
main:
4143
required_status_checks:
@@ -49,6 +51,7 @@ github:
4951
- "test (stable)"
5052
- "test (beta)"
5153
- "test (nightly)"
54+
- "coverage"
5255
- "Release Audit Tool (RAT)"
5356
pull_requests:
5457
# enable updating head branches of pull requests

‎.clusterfuzzlite/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
# under the License.
1717

1818
FROM gcr.io/oss-fuzz-base/base-builder-rust
19+
# pg_query's build script runs bindgen, which needs libclang at build time
20+
RUN apt-get update && apt-get install -y --no-install-recommends libclang-dev && rm -rf /var/lib/apt/lists/*
1921
COPY . $SRC/datafusion-sqlparser-rs
2022
WORKDIR $SRC/datafusion-sqlparser-rs
2123
COPY ./.clusterfuzzlite/build.sh $SRC/

‎.clusterfuzzlite/build.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ set -eu
2020

2121
cd "$SRC/datafusion-sqlparser-rs"
2222
# the base image's RUSTUP_TOOLCHAIN (nightly) overrides the rust-toolchain pin, so we name none
23-
cargo fuzz build -O --fuzz-dir fuzz
23+
# CXXFLAGS in the image carries -stdlib=libc++ while the cc crate defaults to stdc++, so override.
24+
export CXXSTDLIB=c++
25+
cargo fuzz build -O --all-features --fuzz-dir fuzz
2426

2527
targets=$(cargo fuzz list --fuzz-dir fuzz)
2628
if [ -z "$targets" ]; then

‎.github/PULL_REQUEST_TEMPLATE.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
<!-- comma separated, or `none` -->
2+
Dialects:

‎.github/workflows/audit.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ name: Security audit
1919

2020
on:
2121
push:
22-
branches-ignore:
23-
- 'gh-readonly-queue/**'
22+
# Feature branches get CI from pull_request, so push runs only the long-lived branches.
23+
branches:
24+
- main
25+
- 'branch-*'
2426
paths:
2527
- '**/Cargo.toml'
2628
- '**/Cargo.lock'
@@ -37,6 +39,11 @@ on:
3739
permissions:
3840
contents: read
3941

42+
# Only PR runs are disposable. Main, schedule and merge-queue runs keep their verdicts.
43+
concurrency:
44+
group: ${{ github.workflow }}-${{ github.ref }}
45+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
46+
4047
jobs:
4148
audit:
4249
runs-on: ubuntu-latest

‎.github/workflows/labeler.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: labeler
19+
20+
# pull_request_target runs on the base branch, so the script below is never taken from the PR head.
21+
on:
22+
pull_request_target:
23+
types: [opened, edited, reopened, synchronize]
24+
issues:
25+
types: [opened, edited]
26+
issue_comment:
27+
types: [created]
28+
29+
jobs:
30+
label-dialects:
31+
name: Label dialects
32+
# Bot edits and comments arrive in bursts and never carry a dialect.
33+
if: >-
34+
github.event.sender.type != 'Bot' &&
35+
(github.event_name != 'issue_comment' || contains(github.event.comment.body, 'dialect'))
36+
runs-on: ubuntu-latest
37+
permissions:
38+
contents: read
39+
issues: write
40+
pull-requests: write
41+
steps:
42+
- uses: actions/checkout@v4
43+
with:
44+
sparse-checkout: .github/workflows/labeler
45+
persist-credentials: false
46+
- uses: actions/github-script@v9
47+
with:
48+
script: |
49+
const label = require('./.github/workflows/labeler/label_dialects.js');
50+
await label({ github, context });
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
const DIALECTS = [
19+
{ label: "BigQuery", stems: ["bigquery"], pattern: /\bbig\s?query\b/i },
20+
{ label: "ClickHouse", stems: ["clickhouse"], pattern: /\bclick\s?house\b/i },
21+
{ label: "Databricks", stems: ["databricks"], pattern: /\bdatabricks\b/i },
22+
{ label: "DuckDB", stems: ["duckdb"], pattern: /\bduck\s?db\b/i },
23+
{ label: "Hive", stems: ["hive"], pattern: /\bhive\b/i },
24+
{ label: "MySQL", stems: ["mysql"], pattern: /\b(mysql|maria\s?db)\b/i },
25+
{ label: "Oracle", stems: ["oracle"], pattern: /\boracle\b/i },
26+
{ label: "PostgreSQL", stems: ["postgresql", "postgres"], pattern: /\bpostgres/i },
27+
{ label: "Redshift", stems: ["redshift"], pattern: /\bredshift\b/i },
28+
{ label: "Snowflake", stems: ["snowflake"], pattern: /\bsnowflake\b/i },
29+
{ label: "Spark", stems: ["spark"], pattern: /\bspark\b/i },
30+
{ label: "SQL Server", stems: ["mssql"], pattern: /\b(ms\s?sql|sql server|t-?sql)\b/i },
31+
{ label: "SQLite", stems: ["sqlite"], pattern: /\bsqlite\b/i },
32+
{ label: "Teradata", stems: ["teradata"], pattern: /\bteradata\b/i },
33+
];
34+
35+
const DECLARATION = /^[ \t]*dialects?[ \t]*:[ \t]*(.*?)[ \t]*$/im;
36+
const MARKER = "<!-- dialect-labeler -->";
37+
const TRUSTED_ASSOCIATIONS = ["OWNER", "MEMBER", "COLLABORATOR"];
38+
39+
function matchDialects(text) {
40+
return DIALECTS.filter((d) => d.pattern.test(text)).map((d) => d.label);
41+
}
42+
43+
// `undefined` when no `Dialects:` line exists, otherwise the labels it names (possibly none).
44+
function declaredDialects(text) {
45+
const value = DECLARATION.exec(text ?? "")?.[1];
46+
return value ? matchDialects(value) : undefined;
47+
}
48+
49+
function stripMarkup(text) {
50+
return text.replace(/<!--[\s\S]*?-->/g, "").replace(/```[\s\S]*?```/g, "");
51+
}
52+
53+
function resolve({ title, body }) {
54+
body ??= "";
55+
const labels = new Set(matchDialects(title));
56+
const declared = declaredDialects(body);
57+
for (const label of declared ?? []) labels.add(label);
58+
if (labels.size === 0 && declared === undefined) {
59+
for (const label of matchDialects(stripMarkup(body))) labels.add(label);
60+
}
61+
return { labels: [...labels], answered: labels.size > 0 || declared !== undefined };
62+
}
63+
64+
function touchedDialects(paths) {
65+
const stems = new Set();
66+
for (const path of paths) {
67+
const stem = /^src\/dialect\/(\w+)\.rs$|^tests\/sqlparser_(\w+)\.rs$/.exec(path);
68+
if (stem) stems.add(stem[1] ?? stem[2]);
69+
}
70+
return DIALECTS.filter((d) => d.stems.some((s) => stems.has(s))).map((d) => d.label);
71+
}
72+
73+
function isCodeChange(paths) {
74+
return paths.some((path) => path.startsWith("src/") || path.startsWith("tests/"));
75+
}
76+
77+
function askComment(touched) {
78+
const lines = [
79+
MARKER,
80+
"No dialect label was applied because the title does not name a SQL dialect.",
81+
"If this change targets one or more dialects, edit the title to `<Dialect>[, <Dialect>]: <description>`, add a `Dialects: <Dialect>[, <Dialect>]` line to the description, or reply with one. Reply `Dialects: none` if the change is dialect-agnostic.",
82+
];
83+
if (touched.length) lines.push(`Changed files touch ${touched.join(", ")}.`);
84+
return lines.join("\n\n");
85+
}
86+
87+
function trustedComment(comment, issue) {
88+
if (comment.user.type === "Bot") return false;
89+
return comment.user.id === issue.user.id || TRUSTED_ASSOCIATIONS.includes(comment.author_association);
90+
}
91+
92+
async function run({ github, context }) {
93+
const { owner, repo } = context.repo;
94+
const { payload } = context;
95+
const issue = payload.pull_request ?? payload.issue;
96+
const issue_number = issue.number;
97+
let labels = [];
98+
let comment;
99+
100+
if (context.eventName === "issue_comment") {
101+
if (!trustedComment(payload.comment, issue)) return;
102+
labels = declaredDialects(payload.comment.body) ?? [];
103+
} else {
104+
const resolved = resolve(issue);
105+
labels = resolved.labels;
106+
if (!resolved.answered && payload.pull_request && payload.action === "opened") {
107+
const files = await github.paginate(github.rest.pulls.listFiles, {
108+
owner,
109+
repo,
110+
pull_number: issue_number,
111+
per_page: 100,
112+
});
113+
const paths = files.map((f) => f.filename);
114+
if (isCodeChange(paths)) comment = askComment(touchedDialects(paths));
115+
}
116+
}
117+
118+
const existing = new Set(issue.labels.map((l) => l.name));
119+
const missing = labels.filter((l) => !existing.has(l));
120+
if (missing.length) {
121+
await github.rest.issues.addLabels({ owner, repo, issue_number, labels: missing });
122+
}
123+
124+
if (comment) {
125+
const comments = await github.paginate(github.rest.issues.listComments, {
126+
owner,
127+
repo,
128+
issue_number,
129+
per_page: 100,
130+
});
131+
if (comments.some((c) => c.body?.includes(MARKER))) return;
132+
await github.rest.issues.createComment({ owner, repo, issue_number, body: comment });
133+
}
134+
}
135+
136+
module.exports = run;

‎.github/workflows/license.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ on:
2525
pull_request:
2626
merge_group:
2727

28+
# Only PR runs are disposable. Main, schedule and merge-queue runs keep their verdicts.
29+
concurrency:
30+
group: ${{ github.workflow }}-${{ github.ref }}
31+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
32+
2833
jobs:
2934

3035
rat:

‎.github/workflows/rust.yml‎

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,21 @@ name: Rust
1919

2020
on:
2121
push:
22-
# When PR is in the Merge Queue, GitHub will create a temporary branch - but we already have a
23-
# CI running because of `merge_group`
24-
# See also: https://github.com/orgs/community/discussions/15254
25-
branches-ignore:
26-
- 'gh-readonly-queue/**'
22+
# Feature branches get CI from pull_request, so push runs only the long-lived branches.
23+
branches:
24+
- main
25+
- 'branch-*'
2726
pull_request:
2827
merge_group:
2928

3029
permissions:
3130
contents: read
3231

32+
# Only PR runs are disposable. Main, schedule and merge-queue runs keep their verdicts.
33+
concurrency:
34+
group: ${{ github.workflow }}-${{ github.ref }}
35+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
36+
3337
jobs:
3438
codestyle:
3539
runs-on: ubuntu-latest
@@ -45,23 +49,23 @@ jobs:
4549
- uses: actions/checkout@v4
4650
- name: Setup Rust Toolchain
4751
uses: ./.github/actions/setup-builder
48-
- run: cargo clippy --all-targets --all-features -- -D warnings
52+
- run: cargo clippy --locked --all-targets --all-features -- -D warnings
4953

5054
benchmark-lint:
5155
runs-on: ubuntu-latest
5256
steps:
5357
- uses: actions/checkout@v4
5458
- name: Setup Rust Toolchain
5559
uses: ./.github/actions/setup-builder
56-
- run: cd sqlparser_bench && cargo clippy --all-targets --all-features -- -D warnings
60+
- run: cd sqlparser_bench && cargo clippy --locked --all-targets --all-features -- -D warnings
5761

5862
compile:
5963
runs-on: ubuntu-latest
6064
steps:
6165
- uses: actions/checkout@v4
6266
- name: Setup Rust Toolchain
6367
uses: ./.github/actions/setup-builder
64-
- run: cargo check --all-targets --all-features
68+
- run: cargo check --locked --all-targets --all-features
6569

6670
docs:
6771
runs-on: ubuntu-latest
@@ -71,7 +75,7 @@ jobs:
7175
- uses: actions/checkout@v4
7276
- name: Setup Rust Toolchain
7377
uses: ./.github/actions/setup-builder
74-
- run: cargo doc --document-private-items --no-deps --workspace --all-features
78+
- run: cargo doc --locked --document-private-items --no-deps --workspace --all-features
7579

7680
compile-no-std:
7781
runs-on: ubuntu-latest
@@ -81,9 +85,9 @@ jobs:
8185
uses: ./.github/actions/setup-builder
8286
with:
8387
targets: 'thumbv6m-none-eabi'
84-
- run: cargo test --release --no-default-features --test no_std_recursion
85-
- run: cargo check --no-default-features --target thumbv6m-none-eabi
86-
- run: cargo check --no-default-features --features visitor --target thumbv6m-none-eabi
88+
- run: cargo test --locked --release --no-default-features --test no_std_recursion
89+
- run: cargo check --locked --no-default-features --target thumbv6m-none-eabi
90+
- run: cargo check --locked --no-default-features --features visitor --target thumbv6m-none-eabi
8791

8892
test:
8993
strategy:
@@ -98,7 +102,33 @@ jobs:
98102
with:
99103
rust-version: ${{ matrix.rust }}
100104
- uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
101-
- name: Install Tarpaulin
102-
run: cargo install cargo-tarpaulin
103105
- name: Test
104-
run: cargo test --all-features
106+
run: cargo test --locked --all-features
107+
108+
coverage:
109+
runs-on: ubuntu-latest
110+
steps:
111+
- uses: actions/checkout@v4
112+
with:
113+
fetch-depth: 0
114+
- name: Setup Rust Toolchain
115+
uses: ./.github/actions/setup-builder
116+
- run: rustup component add llvm-tools-preview
117+
- uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2.87.19
118+
with:
119+
tool: cargo-llvm-cov
120+
- uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
121+
- run: cargo llvm-cov --locked --all-features --no-report
122+
- run: cargo llvm-cov report --codecov --output-path target/codecov.json
123+
- uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
124+
with:
125+
files: target/codecov.json
126+
fail_ci_if_error: false
127+
token: ${{ secrets.CODECOV_TOKEN }}
128+
- run: cargo llvm-cov report --summary-only --fail-under-lines 90
129+
- name: Changed lines must be covered
130+
if: github.event_name == 'pull_request'
131+
run: |
132+
cargo llvm-cov report --lcov --output-path target/lcov.info
133+
pipx run --spec diff-cover==10.6.0 diff-cover target/lcov.info \
134+
--compare-branch ${{ github.event.pull_request.base.sha }} --fail-under 80

‎.gitignore‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@
55
/derive/target/
66
dev/dist
77

8-
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
9-
# More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock
10-
Cargo.lock
11-
128
# These are backup files generated by rustfmt
139
**/*.rs.bk
1410

0 commit comments

Comments
 (0)