From 13341c6d7217f8ef60573b722cb7e1e4bc436e0a Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 31 Aug 2026 10:45:50 +0900 Subject: [PATCH 1/7] GH-51090: [CI][C++] Fix shellcheck errors in cpp/build-support/fuzzing/generate_corpuses.sh --- .pre-commit-config.yaml | 8 +--- .../fuzzing/generate_corpuses.sh | 48 +++++++++---------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ddf332c57ca..eec8492f8264 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -288,13 +288,7 @@ repos: ( ?^c_glib/.*\.sh$| ?^ci/.*\.sh$| - ?^cpp/build-support/build-lz4-lib\.sh$| - ?^cpp/build-support/build-zstd-lib\.sh$| - ?^cpp/build-support/get-upstream-commit\.sh$| - ?^cpp/build-support/run-test\.sh$| - ?^cpp/build-support/update-flatbuffers\.sh$| - ?^cpp/build-support/update-thrift\.sh$| - ?^cpp/build-support/vendor-flatbuffers\.sh$| + ?^cpp/build-support/.*\.sh$| ?^cpp/examples/minimal_build/run\.sh$| ?^cpp/examples/tutorial_examples/run\.sh$| ?^cpp/src/arrow/flight/sql/odbc/install/mac/postinstall$| diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index 07afa793dc6b..81e4b9827b81 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -29,7 +29,7 @@ set -ex CORPUS_DIR=/tmp/corpus PANDAS_DIR=/tmp/pandas -ARROW_ROOT=$(cd $(dirname "$BASH_SOURCE")/../../..; pwd) +ARROW_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) ARROW_CPP=$ARROW_ROOT/cpp OUT=$1 @@ -42,48 +42,48 @@ OUT=$1 rm -rf ${CORPUS_DIR} ${OUT}/arrow-ipc-generate-fuzz-corpus -stream ${CORPUS_DIR} # Add "golden" IPC integration files -IPC_INTEGRATION_FILES=$(find ${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration -name "*.stream") +IPC_INTEGRATION_FILES=$(find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" -name "*.stream") [ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 # Several IPC integration files can have the same name, make sure # they all appear in the corpus by numbering the duplicates. -cp --backup=numbered ${IPC_INTEGRATION_FILES} ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-stream-fuzz_seed_corpus.zip +cp --backup=numbered "${IPC_INTEGRATION_FILES}" ${CORPUS_DIR} +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" ${CORPUS_DIR} "${OUT}"/arrow-ipc-stream-fuzz_seed_corpus.zip rm -rf ${CORPUS_DIR} -${OUT}/arrow-ipc-generate-fuzz-corpus -file ${CORPUS_DIR} -IPC_INTEGRATION_FILES=$(find ${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration -name "*.arrow_file") +"${OUT}/arrow-ipc-generate-fuzz-corpus" -file "${CORPUS_DIR}" +IPC_INTEGRATION_FILES=$(find "${ARROW_ROOT}"/testing/data/arrow-ipc-stream/integration -name "*.arrow_file") [ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 -cp --backup=numbered ${IPC_INTEGRATION_FILES} ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-file-fuzz_seed_corpus.zip +cp --backup=numbered "${IPC_INTEGRATION_FILES}" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-file-fuzz_seed_corpus.zip -rm -rf ${CORPUS_DIR} -${OUT}/arrow-ipc-generate-tensor-fuzz-corpus -stream ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-tensor-stream-fuzz_seed_corpus.zip +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-ipc-generate-tensor-fuzz-corpus -stream" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-ipc-tensor-stream-fuzz_seed_corpus.zip" # Parquet file-level fuzzer rm -rf ${CORPUS_DIR} -${OUT}/parquet-arrow-generate-fuzz-corpus ${CORPUS_DIR} +"${OUT}/parquet-arrow-generate-fuzz-corpus" "${CORPUS_DIR}" # Add Parquet testing examples -cp ${ARROW_CPP}/submodules/parquet-testing/data/*.parquet ${CORPUS_DIR} -cp ${ARROW_CPP}/submodules/parquet-testing/bad_data/*.parquet ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/parquet-arrow-fuzz_seed_corpus.zip +cp "${ARROW_CPP}/submodules/parquet-testing/data/*.parquet" "${CORPUS_DIR}" +cp "${ARROW_CPP}/submodules/parquet-testing/bad_data/*.parquet" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/parquet-arrow-fuzz_seed_corpus.zip" # Parquet encoding fuzzer rm -rf ${CORPUS_DIR} -${OUT}/parquet-generate-encoding-fuzz-corpus ${CORPUS_DIR} -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/parquet-encoding-fuzz_seed_corpus.zip +"${OUT}/parquet-generate-encoding-fuzz-corpus" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/parquet-encoding-fuzz_seed_corpus.zip" # CSV -rm -rf ${PANDAS_DIR} -git clone --depth=1 https://github.com/pandas-dev/pandas ${PANDAS_DIR} +rm -rf "${PANDAS_DIR}" +git clone --depth=1 https://github.com/pandas-dev/pandas "${PANDAS_DIR}" -rm -rf ${CORPUS_DIR} -${OUT}/arrow-csv-generate-fuzz-corpus ${CORPUS_DIR} +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-csv-generate-fuzz-corpus" "${CORPUS_DIR}" # Add examples from arrow-testing repo -cp ${ARROW_ROOT}/testing/data/csv/*.csv ${CORPUS_DIR} +cp "${ARROW_ROOT}"/testing/data/csv/*.csv "${CORPUS_DIR}" # Add examples from Pandas test suite -find ${PANDAS_DIR}/ -name "*.csv" -exec cp --backup=numbered '{}' ${CORPUS_DIR} \; -${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-csv-fuzz_seed_corpus.zip +find "${PANDAS_DIR}/" -name "*.csv" -exec cp --backup=numbered '{}' "${CORPUS_DIR}" \; +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-csv-fuzz_seed_corpus.zip" From 50a51e3c6d56d1b207d342871b6eb5273d543257 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 31 Aug 2026 10:57:53 +0900 Subject: [PATCH 2/7] GH-51090: [CI][C++] Fix shellcheck errors in cpp/build-support/fuzzing/generate_corpuses.sh --- cpp/build-support/fuzzing/generate_corpuses.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index 81e4b9827b81..9fe28f3912ad 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -39,17 +39,17 @@ OUT=$1 # Arrow IPC -rm -rf ${CORPUS_DIR} -${OUT}/arrow-ipc-generate-fuzz-corpus -stream ${CORPUS_DIR} +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-ipc-generate-fuzz-corpus" -stream "${CORPUS_DIR}" # Add "golden" IPC integration files IPC_INTEGRATION_FILES=$(find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" -name "*.stream") [ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 # Several IPC integration files can have the same name, make sure # they all appear in the corpus by numbering the duplicates. -cp --backup=numbered "${IPC_INTEGRATION_FILES}" ${CORPUS_DIR} -"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" ${CORPUS_DIR} "${OUT}"/arrow-ipc-stream-fuzz_seed_corpus.zip +cp --backup=numbered "${IPC_INTEGRATION_FILES}" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-stream-fuzz_seed_corpus.zip -rm -rf ${CORPUS_DIR} +rm -rf "${CORPUS_DIR}" "${OUT}/arrow-ipc-generate-fuzz-corpus" -file "${CORPUS_DIR}" IPC_INTEGRATION_FILES=$(find "${ARROW_ROOT}"/testing/data/arrow-ipc-stream/integration -name "*.arrow_file") [ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 @@ -62,11 +62,11 @@ rm -rf "${CORPUS_DIR}" # Parquet file-level fuzzer -rm -rf ${CORPUS_DIR} +rm -rf "${CORPUS_DIR}" "${OUT}/parquet-arrow-generate-fuzz-corpus" "${CORPUS_DIR}" # Add Parquet testing examples -cp "${ARROW_CPP}/submodules/parquet-testing/data/*.parquet" "${CORPUS_DIR}" -cp "${ARROW_CPP}/submodules/parquet-testing/bad_data/*.parquet" "${CORPUS_DIR}" +cp "${ARROW_CPP}"/submodules/parquet-testing/data/*.parquet "${CORPUS_DIR}" +cp "${ARROW_CPP}"/submodules/parquet-testing/bad_data/*.parquet "${CORPUS_DIR}" "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/parquet-arrow-fuzz_seed_corpus.zip" # Parquet encoding fuzzer From 34f2e234a899beafba5c43d9edff830d60d24856 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 1 Sep 2026 14:02:01 +0900 Subject: [PATCH 3/7] Use array --- cpp/build-support/fuzzing/generate_corpuses.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index 9fe28f3912ad..d6d1db1d8e83 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -42,18 +42,24 @@ OUT=$1 rm -rf "${CORPUS_DIR}" "${OUT}/arrow-ipc-generate-fuzz-corpus" -stream "${CORPUS_DIR}" # Add "golden" IPC integration files -IPC_INTEGRATION_FILES=$(find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" -name "*.stream") -[ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 +mapfile -d '' IPC_INTEGRATION_FILES < <( + find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ + -name "*.stream" -print0 +) +[ -z "${IPC_INTEGRATION_FILES[@]}" ] && exit 1 # Several IPC integration files can have the same name, make sure # they all appear in the corpus by numbering the duplicates. -cp --backup=numbered "${IPC_INTEGRATION_FILES}" "${CORPUS_DIR}" +cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-stream-fuzz_seed_corpus.zip rm -rf "${CORPUS_DIR}" "${OUT}/arrow-ipc-generate-fuzz-corpus" -file "${CORPUS_DIR}" -IPC_INTEGRATION_FILES=$(find "${ARROW_ROOT}"/testing/data/arrow-ipc-stream/integration -name "*.arrow_file") -[ -z "${IPC_INTEGRATION_FILES}" ] && exit 1 -cp --backup=numbered "${IPC_INTEGRATION_FILES}" "${CORPUS_DIR}" +mapfile -d '' IPC_INTEGRATION_FILES < <( + find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ + -name "*.arrow_file" -print0 +) +[ -z "${IPC_INTEGRATION_FILES[@]}" ] && exit 1 +cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-file-fuzz_seed_corpus.zip rm -rf "${CORPUS_DIR}" From 83df9713ed409405234b94e7d633898ecf900d63 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 1 Sep 2026 14:16:55 +0900 Subject: [PATCH 4/7] Fix SC2198 --- cpp/build-support/fuzzing/generate_corpuses.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index d6d1db1d8e83..1d7a79d402d5 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -46,7 +46,7 @@ mapfile -d '' IPC_INTEGRATION_FILES < <( find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ -name "*.stream" -print0 ) -[ -z "${IPC_INTEGRATION_FILES[@]}" ] && exit 1 +[ "${#IPC_INTEGRATION_FILES[@]}" -eq 0 ] && exit 1 # Several IPC integration files can have the same name, make sure # they all appear in the corpus by numbering the duplicates. cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" @@ -58,7 +58,7 @@ mapfile -d '' IPC_INTEGRATION_FILES < <( find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ -name "*.arrow_file" -print0 ) -[ -z "${IPC_INTEGRATION_FILES[@]}" ] && exit 1 +[ "${#IPC_INTEGRATION_FILES[@]}" -eq 0 ] && exit 1 cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-file-fuzz_seed_corpus.zip From ab82e35bbcb406422e10f48cf57b6d919df19c04 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 1 Sep 2026 14:40:29 +0900 Subject: [PATCH 5/7] fix invalid quote --- cpp/build-support/fuzzing/generate_corpuses.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index 1d7a79d402d5..e18443051257 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -63,7 +63,7 @@ cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-file-fuzz_seed_corpus.zip rm -rf "${CORPUS_DIR}" -"${OUT}/arrow-ipc-generate-tensor-fuzz-corpus -stream" "${CORPUS_DIR}" +"${OUT}/arrow-ipc-generate-tensor-fuzz-corpus" -stream "${CORPUS_DIR}" "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-ipc-tensor-stream-fuzz_seed_corpus.zip" # Parquet file-level fuzzer From 34c03cf52bf09fef49710e9ffd566fe41b83e08b Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 1 Sep 2026 15:32:52 +0900 Subject: [PATCH 6/7] Add comment --- cpp/build-support/fuzzing/generate_corpuses.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index e18443051257..2bc1f74626e4 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -42,10 +42,12 @@ OUT=$1 rm -rf "${CORPUS_DIR}" "${OUT}/arrow-ipc-generate-fuzz-corpus" -stream "${CORPUS_DIR}" # Add "golden" IPC integration files +# Store the files found by the find command in an array. mapfile -d '' IPC_INTEGRATION_FILES < <( find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ -name "*.stream" -print0 ) +# Exit with an error if find returns no files. [ "${#IPC_INTEGRATION_FILES[@]}" -eq 0 ] && exit 1 # Several IPC integration files can have the same name, make sure # they all appear in the corpus by numbering the duplicates. @@ -54,10 +56,12 @@ cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" rm -rf "${CORPUS_DIR}" "${OUT}/arrow-ipc-generate-fuzz-corpus" -file "${CORPUS_DIR}" +# Store the files found by the find command in an array. mapfile -d '' IPC_INTEGRATION_FILES < <( find "${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration" \ -name "*.arrow_file" -print0 ) +# Exit with an error if find returns no files. [ "${#IPC_INTEGRATION_FILES[@]}" -eq 0 ] && exit 1 cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-file-fuzz_seed_corpus.zip From e1922c891aa6dcf9e2b5e7d92df6a42be4b395f6 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 1 Sep 2026 16:08:06 +0900 Subject: [PATCH 7/7] fix quote --- cpp/build-support/fuzzing/generate_corpuses.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index 2bc1f74626e4..38f40a713dfc 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -52,7 +52,7 @@ mapfile -d '' IPC_INTEGRATION_FILES < <( # Several IPC integration files can have the same name, make sure # they all appear in the corpus by numbering the duplicates. cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" -"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-stream-fuzz_seed_corpus.zip +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-ipc-stream-fuzz_seed_corpus.zip" rm -rf "${CORPUS_DIR}" "${OUT}/arrow-ipc-generate-fuzz-corpus" -file "${CORPUS_DIR}" @@ -64,7 +64,7 @@ mapfile -d '' IPC_INTEGRATION_FILES < <( # Exit with an error if find returns no files. [ "${#IPC_INTEGRATION_FILES[@]}" -eq 0 ] && exit 1 cp --backup=numbered "${IPC_INTEGRATION_FILES[@]}" "${CORPUS_DIR}" -"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}"/arrow-ipc-file-fuzz_seed_corpus.zip +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-ipc-file-fuzz_seed_corpus.zip" rm -rf "${CORPUS_DIR}" "${OUT}/arrow-ipc-generate-tensor-fuzz-corpus" -stream "${CORPUS_DIR}"