diff --git a/libaegisub/common/karaoke_matcher.cpp b/libaegisub/common/karaoke_matcher.cpp index b527d2fcb8..ed16306ea6 100644 --- a/libaegisub/common/karaoke_matcher.cpp +++ b/libaegisub/common/karaoke_matcher.cpp @@ -19,26 +19,15 @@ #include "libaegisub/ass/karaoke.h" #include "libaegisub/exception.h" #include "libaegisub/kana_table.h" +#include "libaegisub/unicode.h" #include "libaegisub/util.h" #include #include #include #include -#include -#include namespace { -bool is_whitespace(std::string_view str) { - size_t i = 0; - while (i < str.size()) { - UChar32 c; - U8_NEXT(str.data(), i, str.size(), c); - if (!u_isUWhiteSpace(c)) return false; - } - return true; -} - // strcmp but ignoring case and accents int compare(std::string_view a, std::string_view b) { UErrorCode err = U_ZERO_ERROR; @@ -66,9 +55,9 @@ agi::KaraokeMatchResult agi::AutoMatchKaraoke(std::vector cons // Eat all the whitespace at the beginning of the source and destination // syllables and exit if either ran out. auto eat_whitespace = [&]() -> bool { - while (!src.done() && is_whitespace(src.current())) + while (!src.done() && agi::unicode::is_whitespace(src.current())) src.next(); - while (!dst.done() && is_whitespace(dst.current())) { + while (!dst.done() && agi::unicode::is_whitespace(dst.current())) { dst.next(); ++result.destination_length; } @@ -167,7 +156,7 @@ agi::KaraokeMatchResult agi::AutoMatchKaraoke(std::vector cons size_t src_lookahead_pos = 0; for (auto const& syl : source_strings) { // Don't count blank syllables in the max search distance - if (is_whitespace(syl)) continue; + if (agi::unicode::is_whitespace(syl)) continue; if (++src_lookahead_pos == 1) continue; if (src_lookahead_pos > src_lookahead_max) break; diff --git a/libaegisub/common/unicode.cpp b/libaegisub/common/unicode.cpp index f9508c2def..9106007347 100644 --- a/libaegisub/common/unicode.cpp +++ b/libaegisub/common/unicode.cpp @@ -18,8 +18,21 @@ #include "libaegisub/exception.h" +#include +#include + using namespace agi; +bool agi::unicode::is_whitespace(std::string_view str) { + size_t i = 0; + while (i < str.size()) { + UChar32 c; + U8_NEXT(str.data(), i, str.size(), c); + if (!u_isUWhiteSpace(c)) return false; + } + return true; +} + BreakIterator::BreakIterator() { UErrorCode err = U_ZERO_ERROR; bi.reset(icu::BreakIterator::createCharacterInstance(icu::Locale::getDefault(), err)); diff --git a/libaegisub/include/libaegisub/unicode.h b/libaegisub/include/libaegisub/unicode.h index 817645357c..e24b997078 100644 --- a/libaegisub/include/libaegisub/unicode.h +++ b/libaegisub/include/libaegisub/unicode.h @@ -22,6 +22,11 @@ namespace agi { +namespace unicode { + /// Check if every UTF-8 code point in a string has the Unicode White_Space property + bool is_whitespace(std::string_view str); +} + class BreakIterator { std::unique_ptr bi; std::string_view str; diff --git a/src/dialog_translation.cpp b/src/dialog_translation.cpp index b240589e70..b317793869 100644 --- a/src/dialog_translation.cpp +++ b/src/dialog_translation.cpp @@ -23,6 +23,7 @@ #include "include/aegisub/context.h" #include "include/aegisub/hotkey.h" +#include "libaegisub/unicode.h" #include "ass_dialogue.h" #include "ass_file.h" @@ -38,7 +39,6 @@ #include "selection_controller.h" #include "video_controller.h" -#include #include #include @@ -57,7 +57,7 @@ static void add_hotkey(wxSizer *sizer, wxWindow *parent, const char *command, wx // Skip over override blocks, comments, and whitespace between blocks static bool bad_block(std::unique_ptr &block) { - bool is_whitespace = boost::all(block->GetText(), boost::is_space()); + bool is_whitespace = agi::unicode::is_whitespace(block->GetText()); return block->GetType() != AssBlockType::PLAIN || (is_whitespace && OPT_GET("Tool/Translation Assistant/Skip Whitespace")->GetBool()); } diff --git a/tests/meson.build b/tests/meson.build index 6243187c20..a6dcb3063d 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -43,6 +43,7 @@ tests_src = [ 'tests/thesaurus.cpp', 'tests/time.cpp', 'tests/type_name.cpp', + 'tests/unicode.cpp', 'tests/util.cpp', 'tests/uuencode.cpp', 'tests/vfr.cpp', diff --git a/tests/tests/unicode.cpp b/tests/tests/unicode.cpp new file mode 100644 index 0000000000..d8d884c51e --- /dev/null +++ b/tests/tests/unicode.cpp @@ -0,0 +1,29 @@ +// Copyright (c) 2026 +// +// Permission to use, copy, modify, and distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +#include + +#include + +TEST(lagi_unicode, whitespace) { + EXPECT_TRUE(agi::unicode::is_whitespace("")); + EXPECT_TRUE(agi::unicode::is_whitespace(" \t\r\n")); + EXPECT_TRUE(agi::unicode::is_whitespace("\xC2\xA0")); + EXPECT_TRUE(agi::unicode::is_whitespace("\xE3\x80\x80")); + EXPECT_TRUE(agi::unicode::is_whitespace(" \xC2\xA0\xE3\x80\x80\t")); + + EXPECT_FALSE(agi::unicode::is_whitespace("text")); + EXPECT_FALSE(agi::unicode::is_whitespace(" \xE3\x80\x80text")); + EXPECT_FALSE(agi::unicode::is_whitespace("\xFF")); +}