diff --git a/ci/cloudbuild/dockerfiles/debian-12-arm64.Dockerfile b/ci/cloudbuild/dockerfiles/debian-12-arm64.Dockerfile index ab98e39383..49f9214f61 100644 --- a/ci/cloudbuild/dockerfiles/debian-12-arm64.Dockerfile +++ b/ci/cloudbuild/dockerfiles/debian-12-arm64.Dockerfile @@ -110,10 +110,11 @@ ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH} COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini COPY ./gha/builds/lib/odbcinst.ini /opt/odbc-driver/odbcinst.ini COPY ./gha/builds/lib/lsan.supp /opt/odbc-driver/lsan.supp -COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.googlebigqueryodbc.ini COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini +COPY ./etc/googlebigqueryodbc_utf16.ini /opt/odbc-driver/googlebigqueryodbc_utf16.ini +COPY ./etc/googlebigqueryodbc_utf8.ini /opt/odbc-driver/googlebigqueryodbc_utf8.ini # glibc 2.17 or later RUN echo 'Installing glibc...' diff --git a/ci/cloudbuild/dockerfiles/ubuntu-20.04-release.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-20.04-release.Dockerfile index e2ea6bc065..0f0bcd934c 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-20.04-release.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-20.04-release.Dockerfile @@ -161,7 +161,8 @@ COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini COPY ./gha/builds/lib/odbcinst.ini /opt/odbc-driver/odbcinst.ini COPY ./gha/builds/lib/lsan.supp /opt/odbc-driver/lsan.supp -COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.googlebigqueryodbc.ini COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini +COPY ./etc/googlebigqueryodbc_utf16.ini /opt/odbc-driver/googlebigqueryodbc_utf16.ini +COPY ./etc/googlebigqueryodbc_utf8.ini /opt/odbc-driver/googlebigqueryodbc_utf8.ini diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-full.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-full.Dockerfile index 8b71fa5624..560d8f6ca1 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-full.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-full.Dockerfile @@ -94,10 +94,11 @@ COPY ./etc/vcpkg-version.txt /tmp/vcpkg-version.txt COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini COPY ./gha/builds/lib/odbcinst.ini /opt/odbc-driver/odbcinst.ini COPY ./gha/builds/lib/lsan.supp /opt/odbc-driver/lsan.supp -COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.googlebigqueryodbc.ini COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini +COPY ./etc/googlebigqueryodbc_utf16.ini /opt/odbc-driver/googlebigqueryodbc_utf16.ini +COPY ./etc/googlebigqueryodbc_utf8.ini /opt/odbc-driver/googlebigqueryodbc_utf8.ini # glibc 2.17 or later RUN echo 'Installing glibc...' diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 33e5f46c3f..d541edeeb7 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -133,10 +133,11 @@ COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini COPY ./gha/builds/lib/odbcinst.ini /opt/odbc-driver/odbcinst.ini COPY ./gha/builds/lib/lsan.supp /opt/odbc-driver/lsan.supp -COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.googlebigqueryodbc.ini COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini +COPY ./etc/googlebigqueryodbc_utf16.ini /opt/odbc-driver/googlebigqueryodbc_utf16.ini +COPY ./etc/googlebigqueryodbc_utf8.ini /opt/odbc-driver/googlebigqueryodbc_utf8.ini # glibc 2.17 or later RUN echo 'Installing glibc...' diff --git a/ci/dependencies/driver-manager-setup-google-driver.sh b/ci/dependencies/driver-manager-setup-google-driver.sh index 400e75f188..17ca70f06c 100644 --- a/ci/dependencies/driver-manager-setup-google-driver.sh +++ b/ci/dependencies/driver-manager-setup-google-driver.sh @@ -50,7 +50,9 @@ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib/ export ODBCSYSINI=/opt/odbc-driver export ODBCINI=/opt/odbc-driver/odbc.ini export CPP_BIGQUERY_ODBC_TEST_SERVICE_ACCOUNT_AUTH_KEY=/opt/odbc-driver/connection/key.json -export GOOGLEBIGQUERYODBCINI=/opt/odbc-driver/google.googlebigqueryodbc.ini +export GOOGLEBIGQUERYODBCINI=/opt/odbc-driver/googlebigqueryodbc.ini +export GOOGLEBIGQUERYODBCINI_UTF16=/opt/odbc-driver/googlebigqueryodbc_utf16.ini +export GOOGLEBIGQUERYODBCINI_UTF8=/opt/odbc-driver/googlebigqueryodbc_utf8.ini cd "$CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR" echo '**** ODBC Driver installation END****' diff --git a/ci/etc/googlebigqueryodbc.ini b/ci/etc/googlebigqueryodbc.ini new file mode 100644 index 0000000000..e8c74d7ae9 --- /dev/null +++ b/ci/etc/googlebigqueryodbc.ini @@ -0,0 +1,13 @@ +[Driver] +LogLevel=0 +LogPath= +# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS +# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4). +# +# Accepted values: +# UTF-8 - 1-byte UTF-8 per character +# UTF-16LE - 2-byte UTF-16LE per code unit +# UTF-32LE - 4-byte UTF-32LE per code unit +# (empty) - use sizeof(SQLWCHAR) as-is (default) +# +WcharEncoding= diff --git a/ci/etc/googlebigqueryodbc_utf16.ini b/ci/etc/googlebigqueryodbc_utf16.ini new file mode 100644 index 0000000000..d193389fbe --- /dev/null +++ b/ci/etc/googlebigqueryodbc_utf16.ini @@ -0,0 +1,13 @@ +[Driver] +LogLevel=0 +LogPath= +# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS +# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4). +# +# Accepted values: +# UTF-8 - 1-byte UTF-8 per character +# UTF-16LE - 2-byte UTF-16LE per code unit +# UTF-32LE - 4-byte UTF-32LE per code unit +# (empty) - use sizeof(SQLWCHAR) as-is (default) +# +WcharEncoding=UTF-16LE diff --git a/ci/etc/googlebigqueryodbc_utf8.ini b/ci/etc/googlebigqueryodbc_utf8.ini new file mode 100644 index 0000000000..6c95d3cf78 --- /dev/null +++ b/ci/etc/googlebigqueryodbc_utf8.ini @@ -0,0 +1,13 @@ +[Driver] +LogLevel=0 +LogPath= +# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS +# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4). +# +# Accepted values: +# UTF-8 - 1-byte UTF-8 per character +# UTF-16LE - 2-byte UTF-16LE per code unit +# UTF-32LE - 4-byte UTF-32LE per code unit +# (empty) - use sizeof(SQLWCHAR) as-is (default) +# +WcharEncoding=UTF-8 diff --git a/ci/gha/builds/lib/google.googlebigqueryodbc.ini b/ci/gha/builds/lib/google.googlebigqueryodbc.ini deleted file mode 100644 index cd13ac1435..0000000000 --- a/ci/gha/builds/lib/google.googlebigqueryodbc.ini +++ /dev/null @@ -1,3 +0,0 @@ -[Driver] -LogLevel=0 -LogPath= diff --git a/ci/gha/builds/lib/googlebigqueryodbc.ini b/ci/gha/builds/lib/googlebigqueryodbc.ini new file mode 100644 index 0000000000..e8c74d7ae9 --- /dev/null +++ b/ci/gha/builds/lib/googlebigqueryodbc.ini @@ -0,0 +1,13 @@ +[Driver] +LogLevel=0 +LogPath= +# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS +# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4). +# +# Accepted values: +# UTF-8 - 1-byte UTF-8 per character +# UTF-16LE - 2-byte UTF-16LE per code unit +# UTF-32LE - 4-byte UTF-32LE per code unit +# (empty) - use sizeof(SQLWCHAR) as-is (default) +# +WcharEncoding= diff --git a/ci/gha/builds/macos-cmake.sh b/ci/gha/builds/macos-cmake.sh index b80df3c20b..181de46818 100755 --- a/ci/gha/builds/macos-cmake.sh +++ b/ci/gha/builds/macos-cmake.sh @@ -49,12 +49,16 @@ mv /Users/runner/work/connection/external_account_auth_keys.json.tmp /Users/runn cp ci/gha/builds/lib/odbc_osx.ini /Users/runner/work/connection/odbc-driver/odbc.ini cp ci/gha/builds/lib/odbcinst_osx.ini /Users/runner/work/connection/odbc-driver/odbcinst.ini -cp ci/gha/builds/lib/google.googlebigqueryodbc.ini /Users/runner/work/connection/google.googlebigqueryodbc.ini +cp ci/gha/builds/lib/googlebigqueryodbc.ini /Users/runner/work/connection/googlebigqueryodbc.ini +cp ci/etc/googlebigqueryodbc_utf16.ini /Users/runner/work/connection/googlebigqueryodbc_utf16.ini +cp ci/etc/googlebigqueryodbc_utf8.ini /Users/runner/work/connection/googlebigqueryodbc_utf8.ini # Copy the roots.pem file to the .so directory to run test cases. cp ci/etc/roots.pem /Users/runner/work/cpp-bigquery-odbc/cpp-bigquery-odbc/cmake-out/google/cloud/odbc/roots.pem export ODBCINI=/Users/runner/work/connection/odbc-driver/odbc.ini export ODBCINSTINI=/Users/runner/work/connection/odbc-driver/odbcinst.ini -export GOOGLEBIGQUERYODBCINI=/Users/runner/work/connection/google.googlebigqueryodbc.ini +export GOOGLEBIGQUERYODBCINI=/Users/runner/work/connection/googlebigqueryodbc.ini +export GOOGLEBIGQUERYODBCINI_UTF16=/Users/runner/work/connection/googlebigqueryodbc_utf16.ini +export GOOGLEBIGQUERYODBCINI_UTF8=/Users/runner/work/connection/googlebigqueryodbc_utf8.ini export ODBC_TESTS_DSN="SampleDSNGoogleDriver" export CPP_BIGQUERY_ODBC_TEST_SERVICE_ACCOUNT_AUTH_KEY=/Users/runner/work/connection/key.json export CPP_BIGQUERY_ODBC_TEST_EXTERNAL_ACCOUNT_AUTH_KEY=/Users/runner/work/connection/external_account_auth_keys.json diff --git a/ci/gha/builds/release/googlebigqueryodbc.ini b/ci/gha/builds/release/googlebigqueryodbc.ini index 1c82abe4ae..28807fe0f0 100755 --- a/ci/gha/builds/release/googlebigqueryodbc.ini +++ b/ci/gha/builds/release/googlebigqueryodbc.ini @@ -21,3 +21,14 @@ LogFileSize=2000 # MaxThreads(Optional): Maximum threads the driver will spawn for catalog APIs MaxThreads=8 + +# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS +# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4). +# +# Accepted values: +# UTF-8 - 1-byte UTF-8 per character +# UTF-16LE - 2-byte UTF-16LE per code unit +# UTF-32LE - 4-byte UTF-32LE per code unit +# (empty) - use sizeof(SQLWCHAR) as-is (default) +# +WcharEncoding= diff --git a/google/cloud/odbc/CMakeLists.txt b/google/cloud/odbc/CMakeLists.txt index 57882661fd..ec04725342 100644 --- a/google/cloud/odbc/CMakeLists.txt +++ b/google/cloud/odbc/CMakeLists.txt @@ -159,6 +159,15 @@ else () find_package(google_cloud_cpp_serviceusage REQUIRED) endif () +find_package(re2 CONFIG QUIET) +if (NOT TARGET re2::re2) + if (TARGET re2) + add_library(re2::re2 ALIAS re2) + else () + find_package(re2 REQUIRED) + endif () +endif () + # Restore the original BUILD_SHARED_LIBS value set(BUILD_SHARED_LIBS ${ORIGINAL_BUILD_SHARED_LIBS}) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index eace909d6a..ea8df55eee 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -104,7 +104,7 @@ odbc_internal::StatusRecord ConvertFromNumericDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; WStrToOutputBufferResponse(wstr.GetValue(), dest_data.buf, wchar_capacity, @@ -326,7 +326,7 @@ odbc_internal::StatusRecord ConvertFromStringDSValue(DSValue const& src_dsval, } auto src_len = static_cast(wide_str.length()); - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_str, dest_data.buf, wchar_capacity, src_len, required_chars, @@ -907,7 +907,7 @@ odbc_internal::StatusRecord ConvertFromTimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN required_chars = static_cast(wstr->length()) + 1; return WStrToOutputBufferResponse( wstr.GetValue(), dest_buf, wchar_capacity, k_time_src_len, @@ -1007,30 +1007,22 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - std::wstring wstr_val = wstr.GetValue(); - if (!wstr_val.empty() && wstr_val.back() == L'\0') { - wstr_val.pop_back(); - } - std::vector wstr_data(wstr_val.begin(), wstr_val.end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + size_t const wire_sz = WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / static_cast(wire_sz); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { - *res_len = k_timestamp_src_len * sizeof(SQLWCHAR); + *res_len = static_cast(wstr.GetValue().size() * wire_sz); } - std::memcpy(dest, wstr_data.data(), - (k_timestamp_src_len) * sizeof(SQLWCHAR)); - dest[k_timestamp_src_len] = L'\0'; + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, wstr.GetValue().size(), + /*null_terminate=*/true); } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { - *res_len = wchar_capacity * sizeof(SQLWCHAR); + *res_len = wchar_capacity * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * sizeof(SQLWCHAR)); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, + static_cast(wchar_capacity - 1), + /*null_terminate=*/true); LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1189,29 +1181,21 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - std::wstring wstr_val = wstr.GetValue(); - if (!wstr_val.empty() && wstr_val.back() == L'\0') { - wstr_val.pop_back(); - } - std::vector wstr_data(wstr_val.begin(), wstr_val.end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + size_t const wire_sz = WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / static_cast(wire_sz); if (wchar_capacity > k_datetime_src_len) { if (res_len) { - *res_len = k_datetime_src_len * sizeof(SQLWCHAR); + *res_len = static_cast(wstr.GetValue().size() * wire_sz); } - std::memcpy(dest, wstr_data.data(), - (k_datetime_src_len) * sizeof(SQLWCHAR)); - dest[k_datetime_src_len] = L'\0'; + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, wstr.GetValue().size(), + /*null_terminate=*/true); } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { - *res_len = wchar_capacity * sizeof(SQLWCHAR); + *res_len = wchar_capacity * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * sizeof(SQLWCHAR)); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, + static_cast(wchar_capacity - 1), + /*null_terminate=*/true); LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1398,7 +1382,7 @@ odbc_internal::StatusRecord ConvertFromDateDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "DSValueToWchar Conversion Failed"}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( @@ -1434,7 +1418,7 @@ StatusRecord ConvertStringToJsonOutputBuffer(std::string const& src_str, return StatusRecord{SQLStates::k_HY000(), "Conversion to UTF-16 failed"}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_string.GetValue(), dest_buf, @@ -1498,15 +1482,11 @@ StatusRecord ConvertFromArrayDSValue(DSValue const& src_dsval, if (!wide_string.Ok()) { return StatusRecord{SQLStates::k_HY000(), "Conversion Failed"}; } - std::wstring wide_val = wide_string.GetValue(); - if (!wide_val.empty() && wide_val.back() == L'\0') { - wide_val.pop_back(); - } - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); - auto src_len = static_cast(wide_val.length()); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); + auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( - wide_val, dest_data.buf, wchar_capacity, src_len, required_chars, + *wide_string, dest_data.buf, wchar_capacity, src_len, required_chars, reinterpret_cast(dest_data.result_len)); } case SQL_C_BINARY: { @@ -1621,7 +1601,7 @@ odbc_internal::StatusRecord ConvertFromIntervalDSValue(DSValue const& src_dsval, StatusRecord{SQLStates::k_HY000(), wstr.GetStatusRecord().message}; break; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto interval_char_length = static_cast(wstr.GetValue().length()); return WStrIntervalBufferResponse( @@ -1907,7 +1887,7 @@ StatusRecord ConvertFromGeographyDSValue(DSValue const& src_dsval, } std::memset(dest_data.buf, 0, buffer_length); std::wstring const& wide_str = wstr.GetValue(); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN src_len = static_cast(wide_str.length()); SQLLEN required_chars = src_len + 1; status_record = WStrToOutputBufferResponse( @@ -2054,38 +2034,35 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, "UTF-8 to UTF-16 conversion failed."}; } - std::wstring utf16_value = utf16_str.GetValue(); - if (!utf16_value.empty() && utf16_value.back() == L'\0') { - utf16_value.pop_back(); - } - size_t const required_size = utf16_value.length() * sizeof(SQLWCHAR); + std::wstring const& utf16_value = utf16_str.GetValue(); - auto* buffer = reinterpret_cast(dest_data.buf); - - // Handle truncation if buffer is insufficient - if (dest_data.buflen < required_size) { - size_t num_chars_to_copy = (dest_data.buflen / sizeof(SQLWCHAR)) - 1; - std::memcpy(buffer, utf16_value.data(), - num_chars_to_copy * sizeof(SQLWCHAR)); - buffer[num_chars_to_copy] = L'\0'; + // Narrow wchar_t -> wire encoding directly into the caller's buffer. + // No intermediate vector; WriteWideToWireBuffer is a memcpy when the wire + // SQLWCHAR width matches sizeof(wchar_t) and a per-element narrowing loop + // only on the iODBC-built / unixODBC-loaded path. + size_t const wire_sz = WireWcharSize(); + size_t const src_chars = utf16_value.size(); + size_t const required_size = src_chars * wire_sz; + if (static_cast(dest_data.buflen) < required_size) { + size_t num_chars_to_copy = dest_data.buflen / wire_sz; + if (num_chars_to_copy > 0) { + num_chars_to_copy--; // leave one slot for the null terminator + WriteWideToWireBuffer(utf16_value, dest_data.buf, num_chars_to_copy, + /*null_terminate=*/true); + } if (dest_data.result_len) { - *dest_data.result_len = dest_data.buflen; + *dest_data.result_len = required_size; } LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { - buffer[i] = static_cast(utf16_str.GetValue()[i]); - } - size_t buffer_chars = dest_data.buflen / sizeof(SQLWCHAR); - if (utf16_str.GetValue().size() < buffer_chars) { - buffer[utf16_str.GetValue().size()] = L'\0'; - } - - // Set output length + bool const can_null_terminate = + static_cast(dest_data.buflen) >= required_size + wire_sz; + WriteWideToWireBuffer(utf16_value, dest_data.buf, src_chars, + can_null_terminate); if (dest_data.result_len) { - *dest_data.result_len = utf16_str.GetValue().size() * sizeof(SQLWCHAR); + *dest_data.result_len = required_size; } return status_record; } @@ -2253,7 +2230,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "Conversion to SQL_C_WCHAR failed."}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN src_len = static_cast(wstr->length()); SQLLEN required_chars = src_len + 1; return WStrToOutputBufferResponse( diff --git a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc index 76832234b3..c20ae95e00 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc @@ -53,7 +53,7 @@ StatusRecordOr ConvertFromCharBuffer(DataBuffer& src_data, auto* wchar_buf = static_cast(src_buf); if ((result_len > 0) || (result_len == SQL_NTS)) { if (result_len > 0) { - result_len /= sizeof(SQLWCHAR); + result_len /= WireWcharSize(); } auto utf8_res = BqConvertSQLWCHARToString( wchar_buf, static_cast(result_len)); diff --git a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc index 8052921952..1a28dcb25d 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc @@ -14,6 +14,7 @@ #include "google/cloud/odbc/bq_driver/internal/odbc_desc_attr.h" #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" +#include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/odbc/internal/sql_state_constants.h" #include "google/cloud/odbc/internal/status_record_or.h" #include @@ -388,7 +389,7 @@ StatusRecord DescriptorRecord::SetOctetLength(SQLSMALLINT type, case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: - octet_length = value * sizeof(SQLWCHAR); + octet_length = value * WireWcharSize(); break; case SQL_DECIMAL: case SQL_NUMERIC: diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc index fc445675b5..18a4c39254 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc @@ -356,7 +356,8 @@ StatusRecordOr ProcessTableResults( for (TableFieldSchema const& table_field_schema : bq_table.schema.fields) { // bq_table_column could contain a search pattern character so do a regex // match. - auto column_pattern = BuildRegex(bq_table_column, metadata_id); + std::unique_ptr column_pattern = + BuildRegex(bq_table_column, metadata_id); if (re2::RE2::FullMatch(table_field_schema.name, *column_pattern)) { auto ds_row_status = CreateResultSetDSRow( conn_handle, bq_table.table_reference.project_id, diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc index 70974bf7f2..09b572964f 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc @@ -299,7 +299,7 @@ void ProcessTableResultsHelper(std::string const& column, expected_sql_int_row.ord_pos = (column == "%" || column.empty()) ? 2 : 1; expected_sql_int_row.is_nullable = "NO"; - auto column_pattern = BuildRegex(column, metadata_id); + std::unique_ptr column_pattern = BuildRegex(column, metadata_id); if (!metadata_id && (column.empty() || column == "%")) { ASSERT_EQ(result_set.rows.size(), 2); diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 88e7f6db84..adb278ae54 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -83,7 +83,8 @@ StatusRecordOr> GetFilteredProjectIds( ODBCBQClient& bq_client, std::string const& projects_filter, SQLULEN metadata_id) { std::vector project_ids; - auto filter_regex = BuildRegex(projects_filter, metadata_id); + std::unique_ptr filter_regex = + BuildRegex(projects_filter, metadata_id); // For now, we use default options. // We can set timeout here as needed later. Options options; diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 993fa7374d..a441bf8014 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -42,25 +42,24 @@ SQLRETURN AddressToPointer(SQLPOINTER ptr, SQLPOINTER out_buf, } odbc_internal::StatusRecord WStrIntervalBufferResponse( - std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, + std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - std::vector wstr_data(wstr.begin(), wstr.end()); - wstr_data.emplace_back(L'\0'); + size_t const wire_sz = WireWcharSize(); - auto* dest = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { - *res_len = char_len * sizeof(SQLWCHAR); + *res_len = static_cast(char_len) * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (char_len) * sizeof(SQLWCHAR)); - dest[char_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, static_cast(char_len), + /*null_terminate=*/true); } else if (buffer_length > whole_digits_count) { if (res_len) { - *res_len = buffer_length * sizeof(SQLWCHAR); + *res_len = buffer_length * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, + static_cast(buffer_length - 1), + /*null_terminate=*/true); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 6630ef929f..4667cce7b8 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -15,11 +15,14 @@ #ifndef CPP_BIGQUERY_ODBC_GOOGLE_CLOUD_ODBC_BQ_DRIVER_INTERNAL_ODBC_TYPE_UTILS_H #define CPP_BIGQUERY_ODBC_GOOGLE_CLOUD_ODBC_BQ_DRIVER_INTERNAL_ODBC_TYPE_UTILS_H +#include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/odbc/internal/diagnostic_records.h" #include "google/cloud/odbc/internal/sql_state_constants.h" +#include #include #include #include +#include #include namespace google::cloud::odbc_bq_driver_internal { @@ -175,13 +178,75 @@ SQLRETURN IntValueToOutputBufferResponse(T val, SQLPOINTER buffer_ptr, return SQL_SUCCESS; } +// Writes `count` wide characters from `src` directly into `dest` using the +// current wire encoding. If `null_terminate` is true, writes a NUL terminator +// at index `count`. `dest` must point to caller-owned storage of at least +// `(count + (null_terminate ? 1 : 0)) * WireWcharSize()` bytes. +inline void WriteWideToWireBuffer(std::wstring const& src, void* dest, + size_t count, bool null_terminate = false) { + if (count > src.size()) count = src.size(); + +#if defined(_WIN32) + std::memcpy(dest, src.data(), count * sizeof(SQLWCHAR)); + if (null_terminate) { + auto* d = static_cast(dest); + d[count] = 0; + } +#else + switch (GetEffectiveWireEncoding()) { + case WireEncoding::kUtf32Le: + case WireEncoding::kDefault: { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast( + static_cast >(src[i])); + } + if (null_terminate) { + d[count] = 0; + } + return; + } + case WireEncoding::kUtf16Le: { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast( + static_cast >(src[i])); + } + if (null_terminate) { + d[count] = 0; + } + return; + } + case WireEncoding::kUtf8: { + auto* d = static_cast(dest); + auto utf8_res = Utf16ToUtf8(src.substr(0, count)); + std::string const& utf8_str = utf8_res.Ok() ? *utf8_res : std::string(); + std::memcpy(dest, utf8_str.data(), utf8_str.size()); + if (null_terminate) { + d[utf8_str.size()] = '\0'; + } + return; + } + } +#endif +} + +// Writes a single wire-format NUL terminator (one code unit, 2 or 4 bytes) +// at byte offset `char_index * WireWcharSize()` from `dest`. +inline void WriteWireNul(void* dest, size_t char_index) { + size_t const wire_sz = WireWcharSize(); + std::memset(static_cast(dest) + (char_index * wire_sz), 0, wire_sz); +} + inline odbc_internal::StatusRecord WStrToOutputBufferResponse( - std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, + std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); + size_t const wire_sz = WireWcharSize(); + if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - reinterpret_cast(dest_buf)[0] = L'\0'; + WriteWireNul(dest_buf, 0); } if (res_len) { *res_len = 0; @@ -189,21 +254,17 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } - std::vector wstr_data(wstr.begin(), wstr.end()); - - auto* dest = reinterpret_cast(dest_buf); if (buffer_length > src_len) { if (res_len) { - *res_len = src_len * sizeof(SQLWCHAR); + *res_len = src_len * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (src_len) * sizeof(SQLWCHAR)); - dest[src_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, src_len, /*null_terminate=*/true); } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { - *res_len = buffer_length * sizeof(SQLWCHAR); + *res_len = buffer_length * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, buffer_length - 1, + /*null_terminate=*/true); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { @@ -221,7 +282,7 @@ SQLRETURN AddressToPointer(SQLPOINTER ptr, SQLPOINTER out_buf, SQLSMALLINT* str_len_ptr); odbc_internal::StatusRecord WStrIntervalBufferResponse( - std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, + std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len); } // namespace google::cloud::odbc_bq_driver_internal diff --git a/google/cloud/odbc/bq_driver/internal/trace_utils.cc b/google/cloud/odbc/bq_driver/internal/trace_utils.cc index 9d9ee8aded..f82d01db9f 100644 --- a/google/cloud/odbc/bq_driver/internal/trace_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/trace_utils.cc @@ -32,6 +32,10 @@ static std::once_flag absl_log_init_flag; std::shared_ptr TraceOptions::options_file_ = nullptr; std::mutex TraceOptions::mu_; +odbc_internal::StatusRecordOr> const + kTraceOptsFile = + TraceOptions::CreateTraceOptionsFile(GetOdbcTraceConfigPath()); + #ifdef _WIN32 constexpr char kPathSeparator = '\\'; #else @@ -295,6 +299,10 @@ TraceOptions::CreateTraceOptionsFile( log_file_size = std::strtol(s.second.c_str(), nullptr, 10); } else if (s.first == kMaxThreadsParam) { max_threads = std::stoull(s.second); +#if !defined(_WIN32) + } else if (s.first == kWcharEncoding) { + SetWcharEncodingFromConfig(s.second); +#endif } } diff --git a/google/cloud/odbc/bq_driver/internal/trace_utils.h b/google/cloud/odbc/bq_driver/internal/trace_utils.h index 6124b12495..a3ba8e1554 100644 --- a/google/cloud/odbc/bq_driver/internal/trace_utils.h +++ b/google/cloud/odbc/bq_driver/internal/trace_utils.h @@ -41,6 +41,9 @@ inline std::string const kLogPath = "LogPath"; inline std::string const kLogFileCount = "LogFileCount"; inline std::string const kLogFileSize = "LogFileSize"; inline std::string const kMaxThreadsParam = "MaxThreads"; +// Key controlling the wire encoding of SQLWCHAR buffers on Linux/macOS. +// Accepted values: "UTF-16LE", "UCS-4LE", or empty (auto-detect). +inline std::string const kWcharEncoding = "WcharEncoding"; inline std::uint32_t const kDefaultMaxThreads = 8; inline std::string const kDefaultMaxFiles = "50"; inline std::string const kDefaultMaxSize = "2000"; @@ -189,9 +192,8 @@ std::string GetFormattedMsg(absl::LogEntry const& entry); // Struct types. ///////////////////////////////////////////// -static odbc_internal::StatusRecordOr> const - kTraceOptsFile = - TraceOptions::CreateTraceOptionsFile(GetOdbcTraceConfigPath()); +extern odbc_internal::StatusRecordOr> const + kTraceOptsFile; } // namespace google::cloud::odbc_bq_driver_internal diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index b9fb984502..f6636a3290 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -21,6 +21,7 @@ #include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/internal/getenv.h" #include +#include #include #include #include @@ -39,6 +40,62 @@ bool g_suppress_dropdown = false; using ::google::cloud::odbc_internal::SQLStates; using ::google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; +#if defined(_WIN32) +WireEncoding GetEffectiveWireEncoding() { return WireEncoding::kUtf16Le; } + +size_t WireWcharSize() { return sizeof(SQLWCHAR); } + +void SetWcharEncodingFromConfig(std::string const&) { + // No-op on Windows: SQLWCHAR is always 2-byte UTF-16LE. +} +#else +namespace { +std::atomic g_wire_encoding{WireEncoding::kDefault}; +} // namespace + +WireEncoding GetEffectiveWireEncoding() { + auto configured = g_wire_encoding.load(std::memory_order_relaxed); + if (configured != WireEncoding::kDefault) { + return configured; + } + // Default is based on compile-time SQLWCHAR size + return (sizeof(SQLWCHAR) == 2) ? WireEncoding::kUtf16Le + : WireEncoding::kUtf32Le; +} + +size_t WireWcharSize() { + switch (GetEffectiveWireEncoding()) { + case WireEncoding::kUtf32Le: + case WireEncoding::kDefault: + return 4; + case WireEncoding::kUtf16Le: + return 2; + case WireEncoding::kUtf8: + return 1; + } + return sizeof(SQLWCHAR); +} + +void SetWcharEncodingFromConfig(std::string const& value) { + if (value == "UTF-8" || value == "UTF8") { + g_wire_encoding.store(WireEncoding::kUtf8, std::memory_order_relaxed); + LOG(INFO) << "WcharEncoding: UTF-8 wire format (1 byte/char)"; + } else if (value == "UTF-16LE" || value == "UTF16LE" || value == "UTF-16") { + g_wire_encoding.store(WireEncoding::kUtf16Le, std::memory_order_relaxed); + LOG(INFO) << "WcharEncoding: UTF-16LE wire format (2 bytes/char)"; + } else if (value == "UTF-32LE" || value == "UTF32LE" || value == "UTF-32" || + value == "UCS-4LE") { + g_wire_encoding.store(WireEncoding::kUtf32Le, std::memory_order_relaxed); + LOG(INFO) << "WcharEncoding: UTF-32LE wire format (4 bytes/char)"; + } else if (value.empty() || value == "default") { + g_wire_encoding.store(WireEncoding::kDefault, std::memory_order_relaxed); + LOG(INFO) << "WcharEncoding: default (sizeof(SQLWCHAR) bytes/char)"; + } else { + LOG(WARNING) << "WcharEncoding: unrecognised value '" << value << "'"; + } +} +#endif + #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; @@ -166,7 +223,7 @@ size_t BufferSizeForType(SQLSMALLINT type, size_t requested) { minimum_size = sizeof(SQL_TIMESTAMP_STRUCT); break; case SQL_C_WCHAR: - minimum_size = sizeof(SQLWCHAR); + minimum_size = WireWcharSize(); break; case SQL_C_SBIGINT: minimum_size = sizeof(SQLBIGINT); @@ -671,7 +728,7 @@ std::string GetOdbcTraceConfigPath() { return *path; } // Default to using ~ path directly - return "/etc/google.googlebigqueryodbc.ini"; + return "/etc/googlebigqueryodbc.ini"; #else return k_trace_reg_path; #endif // _WIN32 @@ -777,7 +834,10 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( return StatusRecord{SQLStates::k_HY000(), "Error while converting string to wstring"}; } - utf16Str.push_back(L'\0'); + // MultiByteToWideChar was called with an explicit input length, so + // utf16Length is the character count without a null terminator and + // wstring::size() already reflects the actual character count, + // matching the Linux iconv path behaviour. return utf16Str; #else iconv_t cd = iconv_open(kFromCode.c_str(), "UTF-8"); @@ -809,31 +869,82 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( iconv_close(cd); - // Resize the output string to the actual converted size + // Resize the output string to the actual converted size. No trailing NUL is + // appended: wstring::size() is the character count, matching the Windows + // MultiByteToWideChar path above. Callers own their own NUL termination. utf16str.resize((outbuf - reinterpret_cast(utf16str.data())) / sizeof(wchar_t)); - utf16str.push_back(L'\0'); return utf16str; #endif } odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( - SQLWCHAR* in_str, SQLINTEGER in_str_len) { + SQLWCHAR const* in_str, SQLINTEGER in_str_len) { if (in_str == nullptr) { return StatusRecord{SQLStates::k_HY000(), "in_str string is empty/Null"}; } - if (((in_str != nullptr) && (in_str[0] == '\0'))) { + +#if defined(_WIN32) + if (in_str[0] == '\0') { return std::string(); } - if (in_str_len == SQL_NTS || in_str_len == NULL) { + if (in_str_len == SQL_NTS || in_str_len == 0) { in_str_len = static_cast(std::char_traits::length(in_str)); } - - // Directly create a wide string std::wstring wstr(in_str, in_str + in_str_len); - return Utf16ToUtf8(wstr); +#else + switch (GetEffectiveWireEncoding()) { + case WireEncoding::kUtf32Le: + case WireEncoding::kDefault: { + auto const* utf32 = reinterpret_cast(in_str); + if (utf32[0] == 0) { + return std::string(); + } + SQLINTEGER count = in_str_len; + if (count == SQL_NTS || count == 0) { + count = 0; + while (utf32[count] != 0) ++count; + } + std::wstring wstr; + wstr.reserve(count); + for (SQLINTEGER i = 0; i < count; ++i) { + wstr.push_back(static_cast(utf32[i])); + } + return Utf16ToUtf8(wstr); + } + case WireEncoding::kUtf16Le: { + auto const* utf16 = reinterpret_cast(in_str); + if (utf16[0] == 0) { + return std::string(); + } + SQLINTEGER count = in_str_len; + if (count == SQL_NTS || count == 0) { + count = 0; + while (utf16[count] != 0) ++count; + } + std::wstring wstr; + wstr.reserve(count); + for (SQLINTEGER i = 0; i < count; ++i) { + wstr.push_back(static_cast(utf16[i])); + } + return Utf16ToUtf8(wstr); + } + case WireEncoding::kUtf8: { + auto const* bytes = reinterpret_cast(in_str); + if (bytes[0] == '\0') { + return std::string(); + } + if (in_str_len == SQL_NTS || in_str_len == 0) { + return std::string(bytes); + } + return std::string(bytes, in_str_len); + } + } + + return std::string(); +#endif } bool IsDiagIdentifierString(SQLSMALLINT DiagIdentifier) { diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index a557ce6725..0f328d7ba5 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -224,7 +224,37 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( std::string_view utf_8_str); odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( - SQLWCHAR* in_str, SQLINTEGER in_str_len); + SQLWCHAR const* in_str, SQLINTEGER in_str_len); + +// Returns true when WcharEncoding=UTF-16LE is set in +// Supported wire encodings for SQLWCHAR buffers across the ODBC driver +// boundary. +enum class WireEncoding { + kDefault, // Inferred from build: UTF-16LE on Windows/unixODBC (2 bytes), + // UTF-32LE on iODBC (4 bytes) + kUtf8, // 1 byte per character (UTF-8) + kUtf16Le, // 2 bytes per character (UTF-16LE, e.g. SAP HANA, + // DriverUnicodeType=1) + kUtf32Le // 4 bytes per character (UTF-32LE, e.g. iODBC native) +}; + +// Returns the effective wire encoding in use at runtime. +WireEncoding GetEffectiveWireEncoding(); + +// Apply the WcharEncoding value read from googlebigqueryodbc.ini. Accepted +// values: +// "UTF-8" 1-byte UTF-8 wire format +// "UTF-16LE" 2-byte UTF-16LE wire format +// "UTF-32LE" 4-byte UTF-32LE wire format +// "" / "default" default: based on sizeof(SQLWCHAR) +// No-op on Windows. +void SetWcharEncodingFromConfig(std::string const& value); + +// Bytes per character on the wire between this driver and its caller. +// Returns 1 for UTF-8, 2 for UTF-16LE, 4 for UTF-32LE. +// Use this in arithmetic expressions converting between byte counts and +// character counts on buffers that cross the driver/caller boundary. +size_t WireWcharSize(); std::wstring SQLWcharToWstring(const SQLWCHAR* in_str); diff --git a/google/cloud/odbc/bq_driver/internal/utils_test.cc b/google/cloud/odbc/bq_driver/internal/utils_test.cc index f134d0a0ca..3d1ca3c3ee 100644 --- a/google/cloud/odbc/bq_driver/internal/utils_test.cc +++ b/google/cloud/odbc/bq_driver/internal/utils_test.cc @@ -246,7 +246,7 @@ TEST(GetOdbcTraceConfigPath, GetDefaultPath) { "GOOGLEBIGQUERYODBCINI"); std::string actual = GetOdbcTraceConfigPath(); - EXPECT_EQ(actual, "/etc/google.googlebigqueryodbc.ini"); + EXPECT_EQ(actual, "/etc/googlebigqueryodbc.ini"); google::cloud::odbc_bigquery_client_interface::SetEnv("GOOGLEBIGQUERYODBCINI", home); } diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 92c4bf0cc1..ae6b976593 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -64,6 +64,8 @@ using google::cloud::odbc_bq_driver_internal::IsInfoTypeString; using google::cloud::odbc_bq_driver_internal::StatementHandle; using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; +using google::cloud::odbc_bq_driver_internal::WireWcharSize; +using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; @@ -74,7 +76,6 @@ using ::google::cloud::odbc_bq_driver::HandleLockError; using google::cloud::odbc_bq_driver::ToCharStr; using google::cloud::odbc_bq_driver::ToSqlChar; -using google::cloud::odbc_bq_driver::ToSqlWChar; constexpr int kBufferLength = 4096; @@ -291,7 +292,9 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); + + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, + out_conn_str_len); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -392,11 +395,16 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - std::memset(outConnectionString, '\0', - outConnectionStringBufferLen * sizeof(SQLWCHAR)); - std::memcpy((SQLWCHAR*)outConnectionString, - ToSqlWChar(utf16_out_conn_str->data()), - utf16_out_conn_str->size() * sizeof(SQLWCHAR)); + { + size_t const dest_chars = + static_cast(outConnectionStringBufferLen); + size_t const to_copy = + std::min(utf16_out_conn_str->size(), dest_chars); + // memset zeros the entire dest, which leaves the trailing wire NUL in + // place after we write `to_copy` chars. + std::memset(outConnectionString, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, to_copy); + } } return rc; @@ -543,7 +551,7 @@ SQLRETURN SQL_API SQLConnectW(SQLHDBC connectionHandle, SQLWCHAR* serverName, return utf16_server_name.GetCalculatedReturnCode(); } serverNameLen = utf16_server_name->length(); - std::memcpy(serverName, ToSqlWChar(utf16_server_name->data()), serverNameLen); + WriteWideToWireBuffer(*utf16_server_name, serverName, serverNameLen); if (w_user_name_len > 0) { StatusRecordOr utf16_user_name = Utf8ToUtf16(*utf8_user_name); @@ -551,7 +559,7 @@ SQLRETURN SQL_API SQLConnectW(SQLHDBC connectionHandle, SQLWCHAR* serverName, return utf16_user_name.GetCalculatedReturnCode(); } userNameLen = utf16_user_name->length(); - std::memcpy(userName, ToSqlWChar(utf16_user_name->data()), userNameLen); + WriteWideToWireBuffer(*utf16_user_name, userName, userNameLen); } if (w_auth_str_len > 0) { @@ -560,7 +568,7 @@ SQLRETURN SQL_API SQLConnectW(SQLHDBC connectionHandle, SQLWCHAR* serverName, return utf16_auth_str.GetCalculatedReturnCode(); } authStringLen = utf16_auth_str->length(); - std::memcpy(authString, ToSqlWChar(utf16_auth_str->data()), authStringLen); + WriteWideToWireBuffer(*utf16_auth_str, authString, authStringLen); } return rc; @@ -643,15 +651,14 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_info_val->begin(), - utf16_info_val->end()); - sql_w_str.emplace_back(L'\0'); - std::size_t bytes_available = - static_cast(infoValueBufferLen); - std::size_t bytes_to_copy = - std::min(sql_w_str.size() * sizeof(SQLWCHAR), bytes_available); - - std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(infoValueBufferLen) / wire_sz; + size_t const to_copy = + std::min(utf16_info_val->size(), dest_chars); + bool const can_null_terminate = to_copy < dest_chars; + WriteWideToWireBuffer(*utf16_info_val, infoValue, to_copy, + can_null_terminate); } } else { if (info_val_buffer_len > 0) { @@ -662,7 +669,7 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, } } if (infoValueStringLen) - *infoValueStringLen = info_val_buffer_len * sizeof(SQLWCHAR); + *infoValueStringLen = info_val_buffer_len * WireWcharSize(); return rc; } @@ -806,7 +813,7 @@ SQLRETURN SQL_API SQLSetConnectAttrW(SQLHDBC connectionHandle, ConnectionValueType::kSqlChr) { if (valueStringLen && valueStringLen > 0) { updated_attrib_status = - ConvertSQLPointerToSQLChar(value, valueStringLen / sizeof(SQLWCHAR)); + ConvertSQLPointerToSQLChar(value, valueStringLen / WireWcharSize()); } else { updated_attrib_status = ConvertSQLPointerToSQLChar(value, valueStringLen); } @@ -900,9 +907,10 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, // Handle Unicode conversion of input parameters. // Call to internal common function for SQLGetConnectAttr and // SQLGetConnectAttrW in odbc_connection.h. + SQLINTEGER internal_str_len = 0; rc = ::google::cloud::odbc_bq_driver::SQLGetConnectAttrInternal( - connectionHandle, attribute, updated_attrib_val, valueBufferLen, - valueStringLen); + connectionHandle, attribute, updated_attrib_val, + static_cast(kBufferLength), &internal_str_len); // Handle unicode conversion for attribute string values for output // parameters. if (SQL_SUCCEEDED(rc) && conn_attr.GetAttributeValueType(attribute) == @@ -912,14 +920,18 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - *valueStringLen = - wcslen(updated_out_attr_status->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str( - updated_out_attr_status->c_str(), - updated_out_attr_status->c_str() + *valueStringLen); - sql_w_str.emplace_back(L'\0'); - std::memset(value, '\0', valueBufferLen); - std::memcpy(value, sql_w_str.data(), sql_w_str.size()); + { + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = static_cast(valueBufferLen) / wire_sz; + size_t const to_copy = + std::min(updated_out_attr_status->size(), dest_chars); + if (valueStringLen) { + *valueStringLen = + static_cast(updated_out_attr_status->size() * wire_sz); + } + std::memset(value, '\0', valueBufferLen); + WriteWideToWireBuffer(*updated_out_attr_status, value, to_copy); + } } return rc; @@ -1151,13 +1163,17 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - out_desc_val_string_len = - wcslen(utf16_out_desc_val->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str(utf16_out_desc_val->begin(), - utf16_out_desc_val->end()); - sql_w_str.emplace_back(L'\0'); - std::memset(outDescValue, '\0', outDescValueBufferLen); - std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); + { + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(outDescValueBufferLen) / wire_sz; + size_t const to_copy = + std::min(utf16_out_desc_val->size(), dest_chars); + out_desc_val_string_len = + static_cast(utf16_out_desc_val->size() * wire_sz); + std::memset(outDescValue, '\0', outDescValueBufferLen); + WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, to_copy); + } } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1238,9 +1254,10 @@ SQLRETURN SQL_API SQLGetDescRecW( if (!utf16_name) { return utf16_name.GetCalculatedReturnCode(); } - std::memset(name, '\0', nameBufferLen * sizeof(SQLWCHAR)); - std::memcpy(name, ToSqlWChar(utf16_name->data()), - name_string_len * sizeof(SQLWCHAR)); + size_t const dest_chars = static_cast(nameBufferLen); + size_t const to_copy = std::min(utf16_name->size(), dest_chars); + std::memset(name, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_name, name, to_copy); } if (nameStringLen) *nameStringLen = name_string_len; @@ -1526,11 +1543,8 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_cur_name->begin(), - utf16_cur_name->end()); - sql_w_str.emplace_back(L'\0'); - std::memcpy(cursorName, sql_w_str.data(), - (sql_w_str.size() + 1) * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_cur_name, cursorName, utf16_cur_name->size(), + /*null_terminate=*/true); } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2103,15 +2117,15 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t const bytes_to_copy = - std::min(static_cast(characterAttributeBufferLen), - wstr.size() * sizeof(SQLWCHAR)); - - std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); - if (characterAttributeBufferLen >= sizeof(SQLWCHAR)) { - SQLWCHAR* wchar_buf = static_cast(characterAttribute); - wchar_buf[bytes_to_copy / sizeof(SQLWCHAR)] = 0; - } + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(characterAttributeBufferLen) / wire_sz; + size_t const to_copy = std::min(wstr.size(), dest_chars); + bool const can_null_terminate = + static_cast(characterAttributeBufferLen) >= wire_sz && + to_copy < dest_chars; + WriteWideToWireBuffer(wstr, characterAttribute, to_copy, + can_null_terminate); character_attribute_string_len = static_cast(wstr.size()); } else { @@ -2123,7 +2137,7 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, *characterAttributeStringLen = character_attribute_string_len; #ifdef WIN32 *characterAttributeStringLen = - character_attribute_string_len * sizeof(SQLWCHAR); + character_attribute_string_len * WireWcharSize(); #endif // WIN32 } @@ -2202,9 +2216,13 @@ SQLRETURN SQL_API SQLColAttributesW(SQLHSTMT statementHandle, if (!utf16_character_attribute) { return utf16_character_attribute.GetCalculatedReturnCode(); } - std::memcpy(characterAttribute, - (SQLPOINTER)ToSqlWChar(utf16_character_attribute->data()), - character_attribute_buffer_len); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(character_attribute_buffer_len) / wire_sz; + size_t const to_copy = + std::min(utf16_character_attribute->size(), dest_chars); + WriteWideToWireBuffer(*utf16_character_attribute, characterAttribute, + to_copy); } if (characterAttributeStringLen) *characterAttributeStringLen = character_attribute_buffer_len; @@ -2285,12 +2303,14 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_col_name->begin(), - utf16_col_name->end()); - sql_w_str.emplace_back(L'\0'); - std::memset(columnName, '\0', columnNameBufferLen); - std::memcpy(columnName, sql_w_str.data(), - column_name_string_len * sizeof(SQLWCHAR)); + { + // columnNameBufferLen is in SQLWCHAR characters per ODBC spec. + size_t const dest_chars = static_cast(columnNameBufferLen); + size_t const to_copy = + std::min(utf16_col_name->size(), dest_chars); + std::memset(columnName, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_col_name, columnName, to_copy); + } } if (columnNameLen) { @@ -2484,7 +2504,7 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, // in odbc_diagnostics.h. rc = google::cloud::odbc_bq_driver::SQLGetDiagFieldInternal( handleType, handle, recNumber, diagIdentifier, updated_diag_info, - diagInfoBufferLen, &diag_info_str_len); + static_cast(kBufferLength), &diag_info_str_len); // Handle Unicode conversion of output parameters. if (SQL_SUCCEEDED(rc) && diag_info_str_len > 0) { @@ -2495,13 +2515,16 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - diag_info_str_len = - wcslen(updated_out_diag_info_status->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str( - updated_out_diag_info_status->c_str(), - updated_out_diag_info_status->c_str() + diag_info_str_len); - sql_w_str.emplace_back(L'\0'); - std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); + { + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(diagInfoBufferLen) / wire_sz; + size_t const to_copy = + std::min(updated_out_diag_info_status->size(), dest_chars); + diag_info_str_len = + static_cast(updated_out_diag_info_status->size() * wire_sz); + WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, to_copy); + } } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); @@ -2587,8 +2610,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), - utf16_sql_state->size() * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_sql_state, sqlState, utf16_sql_state->size(), + /*null_terminate=*/true); } if (messageText && message_text_buffer_len > 0) { @@ -2597,9 +2620,14 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_msg_txt) { return utf16_msg_txt.GetCalculatedReturnCode(); } - std::memset(messageText, '\0', messageTextBufferLen); - std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), - utf16_msg_txt->size() * sizeof(SQLWCHAR)); + { + // messageTextBufferLen is in SQLWCHAR characters per ODBC spec. + size_t const dest_chars = static_cast(messageTextBufferLen); + size_t const to_copy = + std::min(utf16_msg_txt->size(), dest_chars); + std::memset(messageText, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_msg_txt, messageText, to_copy); + } } if (messageTextLen) *messageTextLen = message_text_buffer_len; diff --git a/google/cloud/odbc/bq_driver/odbc_environment.cc b/google/cloud/odbc/bq_driver/odbc_environment.cc index 9fd9251047..d2ef122c69 100644 --- a/google/cloud/odbc/bq_driver/odbc_environment.cc +++ b/google/cloud/odbc/bq_driver/odbc_environment.cc @@ -28,6 +28,8 @@ using google::cloud::odbc_internal::StatusRecordOr; SQLRETURN SQLAllocEnvHandle(SQLHANDLE* out_env_handle) { + google::cloud::odbc_bq_driver_internal::TraceOptions::CreateTraceOptionsFile( + google::cloud::odbc_bq_driver_internal::GetOdbcTraceConfigPath()); auto* env_handle = new EnvironmentHandle(); *out_env_handle = env_handle; return SQL_SUCCESS; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_results.cc b/google/cloud/odbc/bq_driver/odbc_sql_results.cc index c9f1bd4e4f..dc800a116c 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_results.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_results.cc @@ -50,6 +50,7 @@ using google::cloud::odbc_bq_driver_internal::StatementHandle; using google::cloud::odbc_bq_driver_internal::StmtStates; using google::cloud::odbc_bq_driver_internal::StringValueToOutputBufferResponse; using google::cloud::odbc_bq_driver_internal::ToSqlPointer; +using google::cloud::odbc_bq_driver_internal::WireWcharSize; using google::cloud::odbc_bq_driver_internal::WriteRowset; using google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; @@ -776,7 +777,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, // 3. If the data fits or is not a variable-length type, return it directly in // the caller’s buffer. SQLLEN target_buff_len = (target_c_type == SQL_C_WCHAR) - ? (target_value_buffer_len / sizeof(SQLWCHAR)) + ? (target_value_buffer_len / WireWcharSize()) : target_value_buffer_len; if (offset == 0) { if ((ds_val.size() > target_buff_len) && @@ -789,7 +790,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, size_t buffer_size = 0; if (target_c_type == SQL_C_WCHAR) { - buffer_size = (ds_val.size() + 1) * sizeof(SQLWCHAR); + buffer_size = (ds_val.size() + 1) * WireWcharSize(); } else { buffer_size = ds_val.size() + 1; } @@ -845,18 +846,18 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, result_set.translated_data.row_offset = offset + target_value_buffer_len; } else if (target_c_type == SQL_C_WCHAR) { auto data_size = result_set.translated_data.data.size(); - auto max_buff_chars = target_value_buffer_len / sizeof(SQLWCHAR); - auto offset_chars = offset / sizeof(SQLWCHAR); + auto max_buff_chars = target_value_buffer_len / WireWcharSize(); + auto offset_chars = offset / WireWcharSize(); auto remain_chars = (data_size > offset_chars) ? (data_size - offset_chars) : 0; auto copy_chars = (remain_chars >= max_buff_chars) ? (max_buff_chars - 1) : remain_chars; std::memcpy(target_value, result_set.translated_data.data.data() + offset, - copy_chars * sizeof(SQLWCHAR)); + copy_chars * WireWcharSize()); reinterpret_cast(target_value)[copy_chars] = 0; result_set.translated_data.row_offset = - offset + (copy_chars * sizeof(SQLWCHAR)); + offset + (copy_chars * WireWcharSize()); } else { std::memcpy(target_value, result_set.translated_data.data.data() + offset, target_value_buffer_len - 1); diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc index 3a32ffe9e3..3052712907 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc @@ -1804,7 +1804,6 @@ TEST(CatalogTest, SQLTables_Filter_DefaultDataset_SchemaNull) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } - #ifdef BQ_DRIVER_INTEGRATION_TESTS // This test case currently crashes with the existing ODBC Driver for BigQuery // v3.1.6.1026. The crash occurs in SQLColumns when schema_name is NULL, diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc index 253438fdca..a6f2b25268 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc @@ -1655,6 +1655,317 @@ TEST(ConnectionTest, CheckTraceLogFileExist) { EXPECT_TRUE(contains_text); } +#if !defined(_WIN32) +#include +#include +#include +#include +#include + +static std::string FindDriverPath() { + if (char const* env_driver = std::getenv("GOOGLE_ODBC_DRIVER_PATH")) { + if (env_driver[0] != '\0' && std::filesystem::exists(env_driver)) { + return env_driver; + } + } + if (char const* odbc_ini = std::getenv("ODBCINI")) { + std::ifstream file(odbc_ini); + std::string line; + while (std::getline(file, line)) { + auto pos = line.find("Driver"); + if (pos != std::string::npos) { + auto eq = line.find('=', pos); + if (eq != std::string::npos) { + std::string path = line.substr(eq + 1); + path.erase(0, path.find_first_not_of(" \t\r\n")); + path.erase(path.find_last_not_of(" \t\r\n") + 1); + if (!path.empty() && std::filesystem::exists(path)) { + return path; + } + } + } + } + } + return "libgoogle_cloud_odbc_bq_driver.so"; +} + +// Verifies that the BigQuery ODBC driver correctly handles UTF-16LE wire +// encoding for SQLWCHAR buffers when WcharEncoding=UTF-16LE is set in +// googlebigqueryodbc.ini. +// +// Background: +// On non-Windows platforms (Linux/macOS), the driver is typically compiled +// against iODBC headers where sizeof(SQLWCHAR) == 4. However, certain calling +// applications and internal ODBC managers (e.g., SAP HANA) use a 2-byte +// SQLWCHAR (UTF-16LE) wire format and directly invoke the driver's exported +// Unicode APIs without Driver Manager translation. +// +// Note on DriverUnicodeType vs. dlopen: +// While unixODBC supports a 'DriverUnicodeType=1' setting in odbcinst.ini to +// negotiate 2-byte UTF-16 wire format, it is specific to unixODBC and is +// ignored by iODBC and by direct driver loaders like SAP HANA. Testing via +// direct dlopen/dlsym ensures the test accurately replicates the real-world +// direct caller scenario across all CI environments (both iODBC and unixODBC). +// +// This test directly dlopens the driver shared library and calls +// SQLDriverConnectW using a 2-byte UTF-16LE connection string buffer to +// validate: +// 1. Default mode (WcharEncoding empty): The driver treats SQLWCHAR as 4-byte +// and fails to parse the 2-byte buffer. +// 2. UTF-16LE override mode (WcharEncoding=UTF-16LE): The driver successfully +// decodes the 2-byte UTF-16LE buffer and connects. +TEST(ConnectionTest, SQLDriverConnectW_Utf16EncodingOverride) { + if (sizeof(SQLWCHAR) != 4) { + GTEST_SKIP() << "WcharEncoding override is only applicable when " + "sizeof(SQLWCHAR) == 4 (iODBC / release build); current " + "sizeof(SQLWCHAR) is " + << sizeof(SQLWCHAR) << "."; + } + + char const* utf16_ini = std::getenv("GOOGLEBIGQUERYODBCINI_UTF16"); + if (!utf16_ini || utf16_ini[0] == '\0') { + GTEST_SKIP() << "GOOGLEBIGQUERYODBCINI_UTF16 is not set; skipping UTF-16 " + "encoding override test."; + } + + std::string driver_path = FindDriverPath(); + void* probe = dlopen(driver_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (!probe) { + probe = dlopen("libgoogle_cloud_odbc_bq_driver.so", RTLD_NOW | RTLD_LOCAL); + } + if (!probe) { + GTEST_SKIP() << "Cannot dlopen BigQuery ODBC driver at " << driver_path + << " (" << dlerror() << "); skipping."; + } + dlclose(probe); + + // Construct a UTF-16LE connection string buffer manually (2 bytes per char). + std::string conn_str = kDefaultConnectionString; + std::vector utf16_conn; + for (char c : conn_str) { + utf16_conn.push_back(static_cast(c)); + } + utf16_conn.push_back(0); // NUL terminator + + auto run_connect_attempt = [&](char const* ini_path) -> bool { + if (ini_path && ini_path[0] != '\0') { + setenv("GOOGLEBIGQUERYODBCINI", ini_path, 1); + } + + void* handle = dlopen(driver_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (!handle) { + handle = + dlopen("libgoogle_cloud_odbc_bq_driver.so", RTLD_NOW | RTLD_LOCAL); + } + if (!handle) { + return false; + } + + auto sql_alloc_handle = + reinterpret_cast( + dlsym(handle, "SQLAllocHandle")); + auto sql_set_env_attr = reinterpret_cast( + dlsym(handle, "SQLSetEnvAttr")); + auto sql_driver_connect_w = reinterpret_cast(dlsym(handle, "SQLDriverConnectW")); + auto sql_disconnect = reinterpret_cast( + dlsym(handle, "SQLDisconnect")); + auto sql_free_handle = + reinterpret_cast( + dlsym(handle, "SQLFreeHandle")); + + if (!sql_alloc_handle || !sql_set_env_attr || !sql_driver_connect_w || + !sql_disconnect || !sql_free_handle) { + dlclose(handle); + return false; + } + + SQLHENV henv = SQL_NULL_HENV; + SQLHDBC hdbc = SQL_NULL_HDBC; + bool connected = false; + + if (sql_alloc_handle(SQL_HANDLE_ENV, nullptr, &henv) == SQL_SUCCESS) { + if (sql_set_env_attr(henv, SQL_ATTR_ODBC_VERSION, + (SQLPOINTER)SQL_OV_ODBC3, 0) == SQL_SUCCESS) { + if (sql_alloc_handle(SQL_HANDLE_DBC, henv, &hdbc) == SQL_SUCCESS) { + SQLWCHAR* in_str = reinterpret_cast(utf16_conn.data()); + SQLRETURN rc = + sql_driver_connect_w(hdbc, nullptr, in_str, SQL_NTS, nullptr, 0, + nullptr, SQL_DRIVER_COMPLETE); + if (rc == SQL_SUCCESS || rc == SQL_SUCCESS_WITH_INFO) { + connected = true; + sql_disconnect(hdbc); + } + sql_free_handle(SQL_HANDLE_DBC, hdbc); + } + } + sql_free_handle(SQL_HANDLE_ENV, henv); + } + dlclose(handle); + return connected; + }; + + // Test Case 1: Connect with default configuration (GOOGLEBIGQUERYODBCINI -> + // should FAIL) + pid_t pid1 = fork(); + ASSERT_NE(pid1, -1); + if (pid1 == 0) { + bool success = run_connect_attempt(nullptr); + // We expect connection to fail; exit 0 on expected failure, 1 on unexpected + // success + std::exit(success ? 1 : 0); + } + + // Test Case 2: Connect with UTF-16LE configuration + // (GOOGLEBIGQUERYODBCINI_UTF16 -> should SUCCEED) + pid_t pid2 = fork(); + ASSERT_NE(pid2, -1); + if (pid2 == 0) { + bool success = run_connect_attempt(utf16_ini); + // We expect connection to succeed; exit 0 on success, 1 on failure + std::exit(success ? 0 : 1); + } + + // Parent process waits for both children + int status1 = 0; + int status2 = 0; + waitpid(pid1, &status1, 0); + waitpid(pid2, &status2, 0); + + EXPECT_TRUE(WIFEXITED(status1) && WEXITSTATUS(status1) == 0) + << "Test Case 1 (Default / No Override) failed: expected connection to " + "fail but it succeeded or crashed."; + EXPECT_TRUE(WIFEXITED(status2) && WEXITSTATUS(status2) == 0) + << "Test Case 2 (UTF-16LE Override) failed: expected connection to " + "succeed but it failed."; +} + +// Validates that when WcharEncoding=UTF-8 is configured, the driver correctly +// parses a 1-byte UTF-8 SQLWCHAR buffer passed across the driver boundary. +TEST(ConnectionTest, SQLDriverConnectW_Utf8EncodingOverride) { + if (sizeof(SQLWCHAR) != 4) { + GTEST_SKIP() << "WcharEncoding override is only applicable when " + "sizeof(SQLWCHAR) == 4 (iODBC / release build); current " + "sizeof(SQLWCHAR) is " + << sizeof(SQLWCHAR) << "."; + } + + char const* utf8_ini = std::getenv("GOOGLEBIGQUERYODBCINI_UTF8"); + if (!utf8_ini || utf8_ini[0] == '\0') { + GTEST_SKIP() << "GOOGLEBIGQUERYODBCINI_UTF8 is not set; skipping UTF-8 " + "encoding override test."; + } + + std::string driver_path = FindDriverPath(); + void* probe = dlopen(driver_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (!probe) { + probe = dlopen("libgoogle_cloud_odbc_bq_driver.so", RTLD_NOW | RTLD_LOCAL); + } + if (!probe) { + GTEST_SKIP() << "Cannot dlopen BigQuery ODBC driver at " << driver_path + << " (" << dlerror() << "); skipping."; + } + dlclose(probe); + + // Construct a UTF-8 connection string buffer (1 byte per char). + std::string conn_str = kDefaultConnectionString; + + auto run_connect_attempt = [&](char const* ini_path) -> bool { + if (ini_path && ini_path[0] != '\0') { + setenv("GOOGLEBIGQUERYODBCINI", ini_path, 1); + } + + void* handle = dlopen(driver_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (!handle) { + handle = + dlopen("libgoogle_cloud_odbc_bq_driver.so", RTLD_NOW | RTLD_LOCAL); + } + if (!handle) { + return false; + } + + auto sql_alloc_handle = + reinterpret_cast( + dlsym(handle, "SQLAllocHandle")); + auto sql_set_env_attr = reinterpret_cast( + dlsym(handle, "SQLSetEnvAttr")); + auto sql_driver_connect_w = reinterpret_cast(dlsym(handle, "SQLDriverConnectW")); + auto sql_disconnect = reinterpret_cast( + dlsym(handle, "SQLDisconnect")); + auto sql_free_handle = + reinterpret_cast( + dlsym(handle, "SQLFreeHandle")); + + if (!sql_alloc_handle || !sql_set_env_attr || !sql_driver_connect_w || + !sql_disconnect || !sql_free_handle) { + dlclose(handle); + return false; + } + + SQLHENV henv = SQL_NULL_HENV; + SQLHDBC hdbc = SQL_NULL_HDBC; + bool connected = false; + + if (sql_alloc_handle(SQL_HANDLE_ENV, nullptr, &henv) == SQL_SUCCESS) { + if (sql_set_env_attr(henv, SQL_ATTR_ODBC_VERSION, + (SQLPOINTER)SQL_OV_ODBC3, 0) == SQL_SUCCESS) { + if (sql_alloc_handle(SQL_HANDLE_DBC, henv, &hdbc) == SQL_SUCCESS) { + SQLWCHAR* in_str = + reinterpret_cast(const_cast(conn_str.data())); + SQLRETURN rc = + sql_driver_connect_w(hdbc, nullptr, in_str, SQL_NTS, nullptr, 0, + nullptr, SQL_DRIVER_COMPLETE); + if (rc == SQL_SUCCESS || rc == SQL_SUCCESS_WITH_INFO) { + connected = true; + sql_disconnect(hdbc); + } + sql_free_handle(SQL_HANDLE_DBC, hdbc); + } + } + sql_free_handle(SQL_HANDLE_ENV, henv); + } + dlclose(handle); + return connected; + }; + + // Test Case 1: Connect with default configuration (GOOGLEBIGQUERYODBCINI -> + // should FAIL because 1-byte chars are parsed as 4-byte chars) + pid_t pid1 = fork(); + ASSERT_NE(pid1, -1); + if (pid1 == 0) { + bool success = run_connect_attempt(nullptr); + std::exit(success ? 1 : 0); + } + + // Test Case 2: Connect with UTF-8 configuration + // (GOOGLEBIGQUERYODBCINI_UTF8 -> should SUCCEED) + pid_t pid2 = fork(); + ASSERT_NE(pid2, -1); + if (pid2 == 0) { + bool success = run_connect_attempt(utf8_ini); + std::exit(success ? 0 : 1); + } + + // Parent process waits for both children + int status1 = 0; + int status2 = 0; + waitpid(pid1, &status1, 0); + waitpid(pid2, &status2, 0); + + EXPECT_TRUE(WIFEXITED(status1) && WEXITSTATUS(status1) == 0) + << "Test Case 1 (Default / No Override) failed: expected connection to " + "fail with UTF-8 buffer but it succeeded or crashed."; + EXPECT_TRUE(WIFEXITED(status2) && WEXITSTATUS(status2) == 0) + << "Test Case 2 (UTF-8 Override) failed: expected connection to " + "succeed but it failed."; +} +#endif // !defined(_WIN32) + #endif // BQ_DRIVER_INTEGRATION_TESTS } // namespace google::cloud::odbc_tests diff --git a/tools/client/googlebigqueryodbc.ini b/tools/client/googlebigqueryodbc.ini index 6d1333e6af..3fc36ce420 100644 --- a/tools/client/googlebigqueryodbc.ini +++ b/tools/client/googlebigqueryodbc.ini @@ -10,3 +10,14 @@ LogPath=/tmp/google_logs LogFileCount=50 LogFileSize=2000 MaxThreads=8 + +# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS +# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4). +# +# Accepted values: +# UTF-8 - 1-byte UTF-8 per character +# UTF-16LE - 2-byte UTF-16LE per code unit +# UTF-32LE - 4-byte UTF-32LE per code unit +# (empty) - use sizeof(SQLWCHAR) as-is (default) +# +WcharEncoding=