Add a token browser extension to IDLE

The TokenBrowser extension adds a Token Browser command to the Tools
menu.  It opens a window listing the Python tokens of the editor
content, or of the selection if there is one, with the token type names
colored as by "python -m tokenize".

There is one browser per editor; invoking the command again refreshes it
and selects the token at the cursor.  Selecting rows highlights the
matching regions in the editor and, while the browser has focus, moves
the editor cursor there; selecting text or moving the cursor in the
editor selects the matching rows.  Double-clicking a row (or pressing
Escape) hides the browser, revealing the editor at the token.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: serhiy-storchaka

Doc/library/idle.rst

@@ -295,6 +295,21 @@ Stack Viewer
 Auto-open Stack Viewer
    Toggle automatically opening the stack viewer on an unhandled exception.
 
+Tools menu (Shell and Editor)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Token Browser
+   Open a window listing the Python tokens of the editor content
+   (or, in the Shell, the current input),
+   or of the selection if there is one.
+   Token type names are colored as by ``python -m tokenize``.
+   Selecting rows highlights the matching regions in the editor
+   and moves the cursor there;
+   selecting text or moving the cursor in the editor
+   selects the matching rows.
+   Double-click a row, or press :kbd:`Escape`,
+   to hide the browser and return to the editor at the token.
+
 Options menu (Shell and Editor)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

Lib/idlelib/News3.txt

@@ -4,6 +4,11 @@ Released on 2026-10-01
 =========================
 
 
+gh-152941: Add a Token Browser to IDLE, opened from the new Tools menu.
+It lists the Python tokens of the editor content, the Shell input, or
+the selection, with token type names colored as by `python -m tokenize`.
+Patch by Serhiy Storchaka and Claude Code.
+
 gh-152745:  When "Run... Customized" with "Restart shell" unchecked
 while Shell is running code, including waiting for an input('prompt:')
 response, just report that the shell is executing instead of

Lib/idlelib/config-extensions.def

@@ -49,6 +49,12 @@ bell= True
 # See config-keys.def for notes on specifying keys and extend.txt for
 # information on creating IDLE extensions.
 
+# A browser for the Python tokens of the editor, opened from the Tools menu.
+[TokenBrowser]
+enable= 1
+[TokenBrowser_cfgBindings]
+token-browser=
+
 # A fake extension for testing and example purposes.  When enabled and
 # invoked, inserts or deletes z-text at beginning of every line.
 [ZzDummy]

Lib/idlelib/editor.py

@@ -425,6 +425,7 @@ def set_line_and_column(self, event=None):
         ("edit", "_Edit"),
         ("format", "F_ormat"),
         ("run", "_Run"),
+        ("tools", "_Tools"),
         ("options", "_Options"),
         ("window", "_Window"),
         ("help", "_Help"),
@@ -1153,6 +1154,7 @@ def get_standard_extension_names(self):
         return idleConf.GetExtensions(editor_only=True)
 
     extfiles = {  # Map built-in config-extension section names to file names.
+        'TokenBrowser': 'tokenbrowser',
         'ZzDummy': 'zzdummy',
         }
 

Lib/idlelib/idle_test/htest.py

@@ -79,6 +79,16 @@
            "Verify x.y.z versions and test each button, including Close.\n "
     }
 
+_token_browser_spec = {
+    'file': 'tokenbrowser',
+    'kwds': {},
+    'msg': "Select rows in the token table and verify the matching regions\n"
+           "are highlighted in the sample editor above.  Select the whole\n"
+           "editor text, or part of it, and press Refresh.\n"
+           "Double-click a row and verify the editor cursor jumps to the\n"
+           "start of that token and the editor gets focus."
+    }
+
 # TODO implement ^\; adding '<Control-Key-\\>' to function does not work.
 _calltip_window_spec = {
     'file': 'calltip_w',

Lib/idlelib/idle_test/test_config.py

@@ -420,11 +420,12 @@ def test_get_extensions(self):
             ''')
         eq = self.assertEqual
         iGE = idleConf.GetExtensions
-        eq(iGE(shell_only=True), [])
-        eq(iGE(), ['ZzDummy'])
-        eq(iGE(editor_only=True), ['ZzDummy'])
-        eq(iGE(active_only=False), ['ZzDummy', 'DISABLE'])
-        eq(iGE(active_only=False, editor_only=True), ['ZzDummy', 'DISABLE'])
+        eq(iGE(shell_only=True), ['TokenBrowser'])
+        eq(iGE(), ['TokenBrowser', 'ZzDummy'])
+        eq(iGE(editor_only=True), ['TokenBrowser', 'ZzDummy'])
+        eq(iGE(active_only=False), ['TokenBrowser', 'ZzDummy', 'DISABLE'])
+        eq(iGE(active_only=False, editor_only=True),
+           ['TokenBrowser', 'ZzDummy', 'DISABLE'])
         userextn.remove_section('ZzDummy')
         userextn.remove_section('DISABLE')
 
@@ -434,7 +435,8 @@ def test_remove_key_bind_names(self):
 
         self.assertCountEqual(
             conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')),
-            ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', 'ZzDummy'])
+            ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch',
+             'TokenBrowser', 'ZzDummy'])
 
     def test_get_extn_name_for_event(self):
         userextn.read_string('''

Lib/idlelib/idle_test/test_mainmenu.py

@@ -11,7 +11,7 @@ class MainMenuTest(unittest.TestCase):
     def test_menudefs(self):
         actual = [item[0] for item in mainmenu.menudefs]
         expect = ['file', 'edit', 'format', 'run', 'shell',
-                  'debug', 'options', 'window', 'help']
+                  'debug', 'tools', 'options', 'window', 'help']
         self.assertEqual(actual, expect)
 
     def test_default_keydefs(self):