From 926bf8b1fbd796bfc587fd431f542779a1cb6f0b Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Wed, 15 Jul 2026 19:17:29 +0800 Subject: [PATCH] Python: extract keywords from non-English text for topic selection _WORD_PATTERN matched only ASCII (`[a-z0-9]...`), so a message written in CJK, Cyrillic or any other non-Latin script produced an empty keyword set. _select_topics returns early on an empty keyword set, so non-English users never had memory topic files loaded automatically. Make the pattern Unicode-aware (`[^\W_][\w-]+`, a letter/digit start plus word chars/hyphen), which is the exact Unicode generalization of the old pattern: English tokenization is unchanged and CJK/Cyrillic text now yields keywords. --- .../core/agent_framework/_harness/_memory.py | 5 ++++- .../core/tests/core/test_harness_memory.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index 04d80cf049..e27abb05d2 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -67,7 +67,10 @@ """ _FILE_HISTORY_ENCODED_SESSION_PREFIX = "~session-" HistoryMessageFilter = Callable[[Message], Message | None] -_WORD_PATTERN = re.compile(r"[a-z0-9][a-z0-9_-]{1,}", flags=re.IGNORECASE) +# Match word-like tokens of >=2 characters. ``[^\W_]`` is a Unicode letter or +# digit (excluding underscore), so CJK/Cyrillic/etc. text yields keywords too +# instead of nothing -- otherwise non-English messages never match topic files. +_WORD_PATTERN = re.compile(r"[^\W_][\w-]+") def _payload_preview(text: str, *, limit: int = 120) -> str: diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index f5b2733e57..b592cbaea2 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -861,3 +861,21 @@ async def test_memory_extraction_propagates_programmer_errors(tmp_path) -> None: context=context, now=datetime(2026, 4, 22, tzinfo=timezone.utc), ) + + +def test_extract_keywords_handles_non_english_text() -> None: + """Keyword extraction yields tokens for non-English input (#6989). + + The pattern used to be ASCII-only, so CJK/Cyrillic messages produced an + empty keyword set and therefore never matched any topic file. + """ + from agent_framework._harness._memory import _extract_keywords # pyright: ignore[reportPrivateUsage] + + cjk = _extract_keywords([Message(role="user", contents=["こんにちは 元気ですか"])]) + cyrillic = _extract_keywords([Message(role="user", contents=["привет мир друзья"])]) + english = _extract_keywords([Message(role="user", contents=["hello world"])]) + + assert cjk == {"こんにちは", "元気ですか"} + assert cyrillic == {"привет", "мир", "друзья"} + # English extraction is unchanged. + assert english == {"hello", "world"}