diff --git a/.jules/bolt.md b/.jules/bolt.md index 341c7c9..457d6b0 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -67,3 +67,6 @@ ## 2025-02-12 - [Fast Path Execution in Directory Traversal and Log Parsing] **Learning:** Checking for string existence (`if "silence_" not in stderr`) before invoking regex matchers provides significant speed improvements when parsing large blocks of text. Similarly, moving expensive I/O operations like `os.path.realpath` inside conditional blocks prevents redundant disk access when configuration (like path exclusions) isn't utilized. **Action:** When working on large text processing or disk operations, verify if early exit conditions or conditional execution can bypass the expensive system or library calls. +## 2024-07-18 - [Optimize tokenization and word stripping with fast path string checks] +**Learning:** Regex execution in Python, even when compiled, is slower than simple string methods. When parsing or tokenizing text to strip non-word characters, prepend a fast-path check using `str.isalnum()` to bypass the regex overhead for purely alphanumeric words, yielding significant performance gains. +**Action:** When tokenizing or processing text using regular expressions to strip non-word characters, prepend a fast-path check using `str.isalnum()` (or similar) to bypass the regex overhead for purely alphanumeric words. diff --git a/summarize.py b/summarize.py index a5b56fe..ad383d9 100644 --- a/summarize.py +++ b/summarize.py @@ -94,7 +94,10 @@ def _content_words(sentence): """ words = [] for raw in sentence.split(): - token = _TOKEN_STRIP_RE.sub("", raw).lower() + if raw.isalnum(): + token = raw.lower() + else: + token = _TOKEN_STRIP_RE.sub("", raw).lower() if token and token not in _STOPWORDS: words.append(token) return words diff --git a/transcript_search.py b/transcript_search.py index 5bb2fac..e25537a 100644 --- a/transcript_search.py +++ b/transcript_search.py @@ -67,6 +67,9 @@ def tokenize(text: str) -> list[str]: Returns: List of lowercase tokens (possibly empty). """ + # Fast path: skip regex if text is purely alphanumeric and spaces + if text.replace(" ", "").isalnum(): + return text.lower().split() return _WORD_RE.findall(text.lower())