diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ddf332c57ca..eec8492f8264 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -288,13 +288,7 @@ repos: ( ?^c_glib/.*\.sh$| ?^ci/.*\.sh$| - ?^cpp/build-support/build-lz4-lib\.sh$| - ?^cpp/build-support/build-zstd-lib\.sh$| - ?^cpp/build-support/get-upstream-commit\.sh$| - ?^cpp/build-support/run-test\.sh$| - ?^cpp/build-support/update-flatbuffers\.sh$| - ?^cpp/build-support/update-thrift\.sh$| - ?^cpp/build-support/vendor-flatbuffers\.sh$| + ?^cpp/build-support/.*\.sh$| ?^cpp/examples/minimal_build/run\.sh$| ?^cpp/examples/tutorial_examples/run\.sh$| ?^cpp/src/arrow/flight/sql/odbc/install/mac/postinstall$| diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index 07afa793dc6b..38f40a713dfc 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -29,7 +29,7 @@ set -ex CORPUS_DIR=/tmp/corpus PANDAS_DIR=/tmp/pandas -ARROW_ROOT=$(cd $(dirname "$BASH_SOURCE")/../../..; pwd) +ARROW_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) ARROW_CPP=$ARROW_ROOT/cpp OUT=$1 @@ -39,51 +39,61 @@ OUT=$1 # Arrow IPC -rm -rf ${CORPUS_DIR} -${OUT}/arrow-ipc-generate-fuzz-corpus -stream ${CORPUS_DIR} +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-ipc-generate-fuzz-corpus" -stream "${CORPUS_DIR}" # Add "golden" IPC integration files -IPC_INTEGRATION_FILES=$(find ${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration -name "*.stream") -[ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 +# Store the files found by the find command in an array. +mapfile -d '' IPC_INTEGRATION_FILES < <( + find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ + -name "*.stream" -print0 +) +# Exit with an error if find returns no files. +[ "${#IPC_INTEGRATION_FILES[@]}" -eq 0 ] && exit 1 # Several IPC integration files can have the same name, make sure # they all appear in the corpus by numbering the duplicates. -cp --backup=numbered ${IPC_INTEGRATION_FILES} ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-stream-fuzz_seed_corpus.zip - -rm -rf ${CORPUS_DIR} -${OUT}/arrow-ipc-generate-fuzz-corpus -file ${CORPUS_DIR} -IPC_INTEGRATION_FILES=$(find ${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration -name "*.arrow_file") -[ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 -cp --backup=numbered ${IPC_INTEGRATION_FILES} ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-file-fuzz_seed_corpus.zip - -rm -rf ${CORPUS_DIR} -${OUT}/arrow-ipc-generate-tensor-fuzz-corpus -stream ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-tensor-stream-fuzz_seed_corpus.zip +cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-ipc-stream-fuzz_seed_corpus.zip" + +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-ipc-generate-fuzz-corpus" -file "${CORPUS_DIR}" +# Store the files found by the find command in an array. +mapfile -d '' IPC_INTEGRATION_FILES < <( + find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ + -name "*.arrow_file" -print0 +) +# Exit with an error if find returns no files. +[ "${#IPC_INTEGRATION_FILES[@]}" -eq 0 ] && exit 1 +cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-ipc-file-fuzz_seed_corpus.zip" + +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-ipc-generate-tensor-fuzz-corpus" -stream "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-ipc-tensor-stream-fuzz_seed_corpus.zip" # Parquet file-level fuzzer -rm -rf ${CORPUS_DIR} -${OUT}/parquet-arrow-generate-fuzz-corpus ${CORPUS_DIR} +rm -rf "${CORPUS_DIR}" +"${OUT}/parquet-arrow-generate-fuzz-corpus" "${CORPUS_DIR}" # Add Parquet testing examples -cp ${ARROW_CPP}/submodules/parquet-testing/data/*.parquet ${CORPUS_DIR} -cp ${ARROW_CPP}/submodules/parquet-testing/bad_data/*.parquet ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/parquet-arrow-fuzz_seed_corpus.zip +cp "${ARROW_CPP}"/submodules/parquet-testing/data/*.parquet "${CORPUS_DIR}" +cp "${ARROW_CPP}"/submodules/parquet-testing/bad_data/*.parquet "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/parquet-arrow-fuzz_seed_corpus.zip" # Parquet encoding fuzzer rm -rf ${CORPUS_DIR} -${OUT}/parquet-generate-encoding-fuzz-corpus ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/parquet-encoding-fuzz_seed_corpus.zip +"${OUT}/parquet-generate-encoding-fuzz-corpus" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/parquet-encoding-fuzz_seed_corpus.zip" # CSV -rm -rf ${PANDAS_DIR} -git clone --depth=1 https://github.com/pandas-dev/pandas ${PANDAS_DIR} +rm -rf "${PANDAS_DIR}" +git clone --depth=1 https://github.com/pandas-dev/pandas "${PANDAS_DIR}" -rm -rf ${CORPUS_DIR} -${OUT}/arrow-csv-generate-fuzz-corpus ${CORPUS_DIR} +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-csv-generate-fuzz-corpus" "${CORPUS_DIR}" # Add examples from arrow-testing repo -cp ${ARROW_ROOT}/testing/data/csv/*.csv ${CORPUS_DIR} +cp "${ARROW_ROOT}"/testing/data/csv/*.csv "${CORPUS_DIR}" # Add examples from Pandas test suite -find ${PANDAS_DIR}/ -name "*.csv" -exec cp --backup=numbered '{}' ${CORPUS_DIR} \; -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-csv-fuzz_seed_corpus.zip +find "${PANDAS_DIR}/" -name "*.csv" -exec cp --backup=numbered '{}' "${CORPUS_DIR}" \; +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-csv-fuzz_seed_corpus.zip"