diff --git a/include/boost/archive/impl/xml_iarchive_impl.ipp b/include/boost/archive/impl/xml_iarchive_impl.ipp index 2c7b04633..67de61da8 100644 --- a/include/boost/archive/impl/xml_iarchive_impl.ipp +++ b/include/boost/archive/impl/xml_iarchive_impl.ipp @@ -9,23 +9,19 @@ // See http://www.boost.org for updates, documentation, and revision history. #include +#include // copy #include // memcpy #include // NULL +#include // back_inserter #if defined(BOOST_NO_STDC_NAMESPACE) -namespace std{ +namespace std{ using ::memcpy; } // namespace std #endif #ifndef BOOST_NO_CWCHAR -#include // mbstate_t and mbrtowc -#if defined(BOOST_NO_STDC_NAMESPACE) -namespace std{ - using ::mbstate_t; - using ::mbrtowc; - } // namespace std -#endif +#include #endif // BOOST_NO_CWCHAR #include // RogueWave and Dinkumware @@ -67,23 +63,14 @@ xml_iarchive_impl::load(std::wstring &ws){ if(NULL != ws.data()) #endif ws.resize(0); - std::mbstate_t mbs = std::mbstate_t(); - const char * start = s.data(); - const char * end = start + s.size(); - while(start < end){ - wchar_t wc; - std::size_t count = std::mbrtowc(&wc, start, end - start, &mbs); - if(count == static_cast(-1)) - boost::serialization::throw_exception( - iterators::dataflow_exception( - iterators::dataflow_exception::invalid_conversion - ) - ); - if(count == static_cast(-2)) - continue; - start += count; - ws += wc; - } + // The text was written as utf8 by mb_from_wchar, so decode it with the + // facet which matches, rather than with whatever the current locale is. + typedef iterators::wchar_from_mb translator; + std::copy( + translator(s.data()), + translator(), + std::back_inserter(ws) + ); } #endif // BOOST_NO_STD_WSTRING @@ -100,24 +87,13 @@ xml_iarchive_impl::load(wchar_t * ws){ ) ); - std::mbstate_t mbs = std::mbstate_t(); - const char * start = s.data(); - const char * end = start + s.size(); - while(start < end){ - wchar_t wc; - std::size_t length = std::mbrtowc(&wc, start, end - start, &mbs); - if(static_cast(-1) == length) - boost::serialization::throw_exception( - iterators::dataflow_exception( - iterators::dataflow_exception::invalid_conversion - ) - ); - if(static_cast(-2) == length) - continue; - - start += length; - *ws++ = wc; - } + // see the comment in the std::wstring overload above + typedef iterators::wchar_from_mb translator; + ws = std::copy( + translator(s.data()), + translator(), + ws + ); *ws = L'\0'; } #endif // BOOST_NO_INTRINSIC_WCHAR_T diff --git a/include/boost/archive/iterators/wchar_from_mb.hpp b/include/boost/archive/iterators/wchar_from_mb.hpp index 294351b48..b2c1e6183 100644 --- a/include/boost/archive/iterators/wchar_from_mb.hpp +++ b/include/boost/archive/iterators/wchar_from_mb.hpp @@ -165,8 +165,7 @@ void wchar_from_mb::drain(){ const typename boost::iterators::iterator_value::type * input_new_start; typename iterator_value::type * next_available; - BOOST_ATTRIBUTE_UNUSED // redundant with ignore_unused below but clarifies intention - std::codecvt_base::result r = m_codecvt_facet.in( + const std::codecvt_base::result r = m_codecvt_facet.in( m_mbs, m_input.m_buffer.begin(), m_input.m_next_available, @@ -175,7 +174,15 @@ void wchar_from_mb::drain(){ m_output.m_buffer.end(), next_available ); - BOOST_ASSERT(std::codecvt_base::ok == r); + if(std::codecvt_base::error == r){ + boost::serialization::throw_exception( + dataflow_exception(dataflow_exception::invalid_conversion) + ); + } + // A partial result is normal here and not an error: the input buffer is + // filled without regard to character boundaries, so it can end in the + // middle of a multibyte character. Its remaining bytes are kept by the + // shift below and decoded once the rest of them have been read. m_output.m_next_available = next_available; m_output.m_next = m_output.m_buffer.begin(); diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index b79fb1576..fee62a270 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -118,6 +118,7 @@ test-suite "serialization" : [ test-bsl-run_files test_valarray ] [ test-bsl-run_files test_variant : A ] [ test-bsl-run_files test_version_value_type ] + [ test-bsl-run_files test_wstring_utf8 ] [ test-bsl-run_files test_vector : A ] [ test-bsl-run_files test_shared_ptr ] [ test-bsl-run_files test_shared_ptr_coexistence ] diff --git a/test/test_wstring_utf8.cpp b/test/test_wstring_utf8.cpp new file mode 100644 index 000000000..15418c51d --- /dev/null +++ b/test/test_wstring_utf8.cpp @@ -0,0 +1,115 @@ +/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 +// test_wstring_utf8.cpp + +// Copyright 2026 Gennaro Prota. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org for updates, documentation, and revision history. + +// A std::wstring holding characters outside ASCII has to survive a round +// trip. The XML archives are the ones which have to encode it, since xml +// is text, and both ends have to agree on UTF-8. + +// Reported by schorsch1976 in +// https://github.com/boostorg/serialization/issues/298, with a reproducer +// showing that only the xml archive was affected while text and binary, +// which write the characters unchanged, were not. Thanks! + +// The characters are written as universal character names on purpose, so +// that the test does not depend on how the compiler reads this file. + +#include +#include +#include +#include + +#include +#if defined(BOOST_NO_STDC_NAMESPACE) +namespace std{ + using ::remove; +} +#endif + +#include +#include + +#include "test_tools.hpp" + +#ifndef BOOST_NO_STD_WSTRING + +struct data +{ + friend class boost::serialization::access; + template + void serialize(Archive & ar, const unsigned int /* file_version */){ + ar & BOOST_SERIALIZATION_NVP(value); + ar & BOOST_SERIALIZATION_NVP(tail); + } + std::wstring value; + int tail; +}; + +static void round_trip(const char * testfile, const std::wstring & w){ + { + data d; + d.value = w; + d.tail = 42; + test_ostream os(testfile, TEST_STREAM_FLAGS); + test_oarchive oa(os, TEST_ARCHIVE_FLAGS); + oa << BOOST_SERIALIZATION_NVP(d); + } + data d; + d.tail = 0; + test_istream is(testfile, TEST_STREAM_FLAGS); + test_iarchive ia(is, TEST_ARCHIVE_FLAGS); + ia >> BOOST_SERIALIZATION_NVP(d); + BOOST_CHECK(d.value == w); + // whatever follows the string has to be readable as well + BOOST_CHECK(42 == d.tail); +} + +int +test_main(int /* argc */, char * /* argv */ []) +{ + const char * testfile = boost::archive::tmpnam(NULL); + BOOST_REQUIRE(NULL != testfile); + + // two bytes and three bytes each once encoded as UTF-8 + const std::wstring two(L"\u00E9\u00DF"); + const std::wstring three(L"\u4F60\u597D"); + + round_trip(testfile, L""); + round_trip(testfile, L"plain ascii"); + round_trip(testfile, two); + round_trip(testfile, three); + round_trip(testfile, L"mixed " + two + L" and " + three + L" text"); + + // The decoder fills a 32 byte buffer without regard to character + // boundaries, so walk a multibyte character across that boundary. + for(std::size_t pad = 26; pad <= 38; ++pad){ + round_trip(testfile, std::wstring(pad, L'x') + three); + } + + // longer than the buffer, so that it is filled repeatedly + std::wstring big; + for(int i = 0; i < 40; ++i){ + big += three; + big += L"ascii"; + } + round_trip(testfile, big); + + std::remove(testfile); + return EXIT_SUCCESS; +} + +#else // BOOST_NO_STD_WSTRING + +int +test_main(int /* argc */, char * /* argv */ []) +{ + return EXIT_SUCCESS; +} + +#endif // BOOST_NO_STD_WSTRING