Fix Unicode whitespace handling in translation assistant

libaegisub/common/karaoke_matcher.cpp

@@ -19,26 +19,15 @@
 #include "libaegisub/ass/karaoke.h"
 #include "libaegisub/exception.h"
 #include "libaegisub/kana_table.h"
+#include "libaegisub/unicode.h"
 #include "libaegisub/util.h"
 
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/range/algorithm/copy.hpp>
 #include <unicode/brkiter.h>
 #include <unicode/coll.h>
-#include <unicode/uchar.h>
-#include <unicode/utf8.h>
 
 namespace {
-bool is_whitespace(std::string_view str) {
-	size_t i = 0;
-	while (i < str.size()) {
-		UChar32 c;
-		U8_NEXT(str.data(), i, str.size(), c);
-		if (!u_isUWhiteSpace(c)) return false;
-	}
-	return true;
-}
-
 // strcmp but ignoring case and accents
 int compare(std::string_view a, std::string_view b) {
 	UErrorCode err = U_ZERO_ERROR;
@@ -66,9 +55,9 @@ agi::KaraokeMatchResult agi::AutoMatchKaraoke(std::vector<std::string_view> cons
 	// Eat all the whitespace at the beginning of the source and destination
 	// syllables and exit if either ran out.
 	auto eat_whitespace = [&]() -> bool {
-		while (!src.done() && is_whitespace(src.current()))
+		while (!src.done() && agi::unicode::is_whitespace(src.current()))
 			src.next();
-		while (!dst.done() && is_whitespace(dst.current())) {
+		while (!dst.done() && agi::unicode::is_whitespace(dst.current())) {
 			dst.next();
 			++result.destination_length;
 		}
@@ -167,7 +156,7 @@ agi::KaraokeMatchResult agi::AutoMatchKaraoke(std::vector<std::string_view> cons
 		size_t src_lookahead_pos = 0;
 		for (auto const& syl : source_strings) {
 			// Don't count blank syllables in the max search distance
-			if (is_whitespace(syl)) continue;
+			if (agi::unicode::is_whitespace(syl)) continue;
 			if (++src_lookahead_pos == 1) continue;
 			if (src_lookahead_pos > src_lookahead_max) break;
 

libaegisub/common/unicode.cpp

@@ -18,8 +18,21 @@
 
 #include "libaegisub/exception.h"
 
+#include <unicode/uchar.h>
+#include <unicode/utf8.h>
+
 using namespace agi;
 
+bool agi::unicode::is_whitespace(std::string_view str) {
+	size_t i = 0;
+	while (i < str.size()) {
+		UChar32 c;
+		U8_NEXT(str.data(), i, str.size(), c);
+		if (!u_isUWhiteSpace(c)) return false;
+	}
+	return true;
+}
+
 BreakIterator::BreakIterator() {
 	UErrorCode err = U_ZERO_ERROR;
 	bi.reset(icu::BreakIterator::createCharacterInstance(icu::Locale::getDefault(), err));

libaegisub/include/libaegisub/unicode.h

@@ -22,6 +22,11 @@
 
 namespace agi {
 
+namespace unicode {
+	/// Check if every UTF-8 code point in a string has the Unicode White_Space property
+	bool is_whitespace(std::string_view str);
+}
+
 class BreakIterator {
 	std::unique_ptr<icu::BreakIterator> bi;
 	std::string_view str;

src/dialog_translation.cpp

@@ -23,6 +23,7 @@
 
 #include "include/aegisub/context.h"
 #include "include/aegisub/hotkey.h"
+#include "libaegisub/unicode.h"
 
 #include "ass_dialogue.h"
 #include "ass_file.h"
@@ -38,7 +39,6 @@
 #include "selection_controller.h"
 #include "video_controller.h"
 
-#include <boost/algorithm/string/classification.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/replace.hpp>
 
@@ -57,7 +57,7 @@ static void add_hotkey(wxSizer *sizer, wxWindow *parent, const char *command, wx
 
 // Skip over override blocks, comments, and whitespace between blocks
 static bool bad_block(std::unique_ptr<AssDialogueBlock> &block) {
-	bool is_whitespace = boost::all(block->GetText(), boost::is_space());
+	bool is_whitespace = agi::unicode::is_whitespace(block->GetText());
 	return block->GetType() != AssBlockType::PLAIN || (is_whitespace && OPT_GET("Tool/Translation Assistant/Skip Whitespace")->GetBool());
 }
 

tests/meson.build

@@ -43,6 +43,7 @@ tests_src = [
     'tests/thesaurus.cpp',
     'tests/time.cpp',
     'tests/type_name.cpp',
+    'tests/unicode.cpp',
     'tests/util.cpp',
     'tests/uuencode.cpp',
     'tests/vfr.cpp',

tests/tests/unicode.cpp

@@ -0,0 +1,29 @@
+// Copyright (c) 2026
+//
+// Permission to use, copy, modify, and distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+#include <libaegisub/unicode.h>
+
+#include <main.h>
+
+TEST(lagi_unicode, whitespace) {
+	EXPECT_TRUE(agi::unicode::is_whitespace(""));
+	EXPECT_TRUE(agi::unicode::is_whitespace(" \t\r\n"));
+	EXPECT_TRUE(agi::unicode::is_whitespace("\xC2\xA0"));
+	EXPECT_TRUE(agi::unicode::is_whitespace("\xE3\x80\x80"));
+	EXPECT_TRUE(agi::unicode::is_whitespace(" \xC2\xA0\xE3\x80\x80\t"));
+
+	EXPECT_FALSE(agi::unicode::is_whitespace("text"));
+	EXPECT_FALSE(agi::unicode::is_whitespace(" \xE3\x80\x80text"));
+	EXPECT_FALSE(agi::unicode::is_whitespace("\xFF"));
+}