diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 0000000000..579c99ad51 --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM gcr.io/oss-fuzz-base/base-builder-rust +COPY . $SRC/datafusion-sqlparser-rs +WORKDIR $SRC/datafusion-sqlparser-rs +COPY ./.clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100755 index 0000000000..a38f94478d --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +cd "$SRC/datafusion-sqlparser-rs" +# the base image's RUSTUP_TOOLCHAIN (nightly) overrides the rust-toolchain pin, so we name none +cargo fuzz build -O --fuzz-dir fuzz + +targets=$(cargo fuzz list --fuzz-dir fuzz) +if [ -z "$targets" ]; then + echo "cargo fuzz list named no target" >&2 + exit 1 +fi + +target_dir=fuzz/target/x86_64-unknown-linux-gnu/release +for name in $targets; do + cp "$target_dir/$name" "$OUT/" +done diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml new file mode 100644 index 0000000000..c4b81466d2 --- /dev/null +++ b/.clusterfuzzlite/project.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +language: rust +sanitizers: + - address diff --git a/.github/workflows/cflite_batch.yml b/.github/workflows/cflite_batch.yml new file mode 100644 index 0000000000..7694ae91b3 --- /dev/null +++ b/.github/workflows/cflite_batch.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ClusterFuzzLite batch fuzzing +on: + schedule: + - cron: '0 06 * * *' # 06:00 UTC daily +permissions: read-all +jobs: + BatchFuzzing: + runs-on: ubuntu-latest + permissions: + contents: write + strategy: + fail-fast: false + matrix: + sanitizer: + - address + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + language: rust + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 3600 + mode: 'batch' + sanitizer: ${{ matrix.sanitizer }} + storage-repo: ${{ github.server_url }}/${{ github.repository }}.git + storage-repo-branch: clusterfuzzlite diff --git a/.github/workflows/cflite_build.yml b/.github/workflows/cflite_build.yml new file mode 100644 index 0000000000..7afbf075e4 --- /dev/null +++ b/.github/workflows/cflite_build.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ClusterFuzzLite continuous builds +on: + push: + branches: + - main +permissions: read-all +jobs: + Build: + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ matrix.sanitizer }}-${{ github.ref }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + sanitizer: + - address + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + language: rust + sanitizer: ${{ matrix.sanitizer }} + upload-build: true diff --git a/.github/workflows/cflite_pr.yml b/.github/workflows/cflite_pr.yml new file mode 100644 index 0000000000..d6480aeb95 --- /dev/null +++ b/.github/workflows/cflite_pr.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ClusterFuzzLite PR fuzzing +on: + pull_request: +permissions: read-all +jobs: + PR: + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ matrix.sanitizer }}-${{ github.ref }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + sanitizer: + - address + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + language: rust + github-token: ${{ secrets.GITHUB_TOKEN }} + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 600 + mode: 'code-change' + sanitizer: ${{ matrix.sanitizer }} diff --git a/.github/workflows/cflite_prune.yml b/.github/workflows/cflite_prune.yml new file mode 100644 index 0000000000..f125eca705 --- /dev/null +++ b/.github/workflows/cflite_prune.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ClusterFuzzLite corpus pruning +on: + schedule: + - cron: '0 05 * * *' # 05:00 UTC, an hour before batch fuzzing +permissions: read-all +jobs: + Pruning: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Build Fuzzers + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + language: rust + - name: Run Fuzzers + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 600 + mode: 'prune' + storage-repo: ${{ github.server_url }}/${{ github.repository }}.git + storage-repo-branch: clusterfuzzlite diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 691481eab0..3d16551061 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -27,4 +27,8 @@ cd fuzz cargo +nightly fuzz run fuzz_parse_sql -- -max_total_time=600 ``` +ClusterFuzzLite runs continuous fuzzing. Every pull request fuzzes for 10 minutes in +`code-change` mode, a daily batch job grows the shared corpus stored on the +`clusterfuzzlite` branch, and a daily prune compacts it. + Crashes land in `artifacts//` and replay with `cargo fuzz run `.