diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index c7c30e5300c2a4..8ee6a08353f80d 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -295,6 +295,48 @@ Stack Viewer Auto-open Stack Viewer Toggle automatically opening the stack viewer on an unhandled exception. +Browse menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Token Browser + Open a window listing the Python tokens of the editor content + (or, in the Shell, the current input), + or of the selection if there is one. + Token type names are colored as by ``python -m tokenize``. + Selecting rows highlights the matching regions in the editor + and moves the cursor there; + selecting text or moving the cursor in the editor + selects the matching rows. + Double-click a row, or press :kbd:`Escape`, + to hide the browser and return to the editor at the token. + +AST Browser + Open a window showing the abstract syntax tree of the editor content + (or, in the Shell, the current input), + or of the selection if there is one. + Selecting a node highlights the matching region in the editor + and moves the cursor there; + selecting text or moving the cursor in the editor + selects the innermost enclosing node. + Double-click a node, or press :kbd:`Escape`, + to hide the browser and return to the editor at the node. + +Disassembly Browser + Open a window showing the disassembled bytecode of the editor content + (or, in the Shell, the current input), or of the selection if there is one. + Each code object is a collapsible row of its instructions, + colored by the kind of operand they use; + the one holding the cursor is opened and its instructions selected. + Selecting an instruction highlights the matching region in the editor + and moves the cursor there; + selecting text or moving the cursor in the editor + selects the instructions built from it. + Double-click a row, or press :kbd:`Escape`, + to hide the browser and return to the editor at the instruction. + While the debugger is stopped, the browser follows it instead of the + editor, showing the code object that is executing + and marking the current instruction. + Options menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index bcbd7fa542a02a..5f1431f6feaa2b 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,26 @@ Released on 2026-10-01 ========================= +gh-152966: Add a Disassembly Browser to IDLE, opened from the Browse +menu. It shows the disassembled bytecode of the editor content, the +Shell input, or the selection, as collapsible code objects colored by +operand kind. Selecting an instruction highlights the matching source, +and selecting source selects the instructions. While the debugger is +stopped, the browser instead shows the code object that is executing and +marks the current instruction. Patch by Serhiy Storchaka and Claude +Code. + +gh-152942: Add an AST Browser to IDLE, opened from the Browse menu. It +shows the abstract syntax tree of the editor content, the Shell input, +or the selection. Selecting a node highlights the matching region in +the editor, and selecting text in the editor selects the innermost +enclosing node. Patch by Serhiy Storchaka and Claude Code. + +gh-152941: Add a Token Browser to IDLE, opened from the new Browse menu. +It lists the Python tokens of the editor content, the Shell input, or +the selection, with token type names colored as by `python -m tokenize`. +Patch by Serhiy Storchaka and Claude Code. + gh-152745: When "Run... Customized" with "Restart shell" unchecked while Shell is running code, including waiting for an input('prompt:') response, just report that the shell is executing instead of diff --git a/Lib/idlelib/astbrowser.py b/Lib/idlelib/astbrowser.py new file mode 100644 index 00000000000000..92ca7cc2e2aaf5 --- /dev/null +++ b/Lib/idlelib/astbrowser.py @@ -0,0 +1,318 @@ +"""An AST browser for IDLE. + +The Browse menu's "AST Browser" command (see open() below) opens a window +showing the abstract syntax tree of the editor content (or, in the Shell, +the current input), or of the selection if there is one. Selecting a node +highlights the matching region in the editor and moves the editor cursor +there; selecting text or moving the cursor in the editor selects the +innermost matching node. Double-clicking a node hides the browser (as +does Escape), revealing the editor at the node. +""" +import ast + +from tkinter import Toplevel, TclError +from tkinter import TOP, BOTTOM, LEFT, RIGHT, X, Y, BOTH, W, END, VERTICAL +from tkinter import ttk + +from idlelib.config import idleConf + +# The editor tag that highlights the source of the selected nodes. +TAG = "ASTBROWSER" + + +def open(editwin): + "Open the AST browser for editwin, reusing one already open." + window = getattr(editwin, "ast_browser", None) + if window is not None and window.winfo_exists(): + window.refresh() + else: + editwin.ast_browser = ASTBrowserWindow(editwin.top, editwin.text) + + +class ASTBrowserWindow(Toplevel): + "Show the abstract syntax tree of a Text widget's content or selection." + + def __init__(self, parent, text, *, _htest=False, _utest=False): + """Create the AST browser. + + parent - the master widget of this window. + text - the editor Text widget to browse and drive. + _htest - bool; change box location when running htest. + _utest - bool; don't wait for user interaction when unit testing. + """ + super().__init__(parent) + self.text = text + self.base = (1, 0) # Editor index of the parsed region's start. + self.source_lines = [] # Lines of the parsed source (for byte->char). + self.ranges = {} # Tree item id -> (start index, end index). + self.focused = False # Whether the browser currently has the focus. + self.title("AST Browser") + self.protocol("WM_DELETE_WINDOW", self.hide) + self.bind("", self.hide) + x = parent.winfo_rootx() + 20 + y = parent.winfo_rooty() + (100 if _htest else 20) + self.geometry(f"640x480+{x}+{y}") + self.minsize(400, 300) + + self.create_widgets() + self.configure_tag() + self.populate() + # Follow the editor and select the matching node. <> covers + # selection changes by keyboard or mouse (a generic is + # shadowed by IDLE's specific key bindings); the release events cover + # plain cursor moves that leave no selection. These bindings live as + # long as the editor Text and are torn down together with it (and with + # this child window), so there is nothing to unbind. + text.bind("<>", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + if not _utest: + self.deiconify() + + def create_widgets(self): + bar = ttk.Frame(self, padding=(6, 6, 6, 0)) + bar.pack(side=TOP, fill=X) + ttk.Button(bar, text="Refresh", command=self.populate).pack(side=LEFT) + + self.status = ttk.Label(self, anchor=W, relief="sunken", padding=3) + self.status.pack(side=BOTTOM, fill=X) + + frame = ttk.Frame(self, padding=6) + frame.pack(side=TOP, fill=BOTH, expand=True) + self.tree = ttk.Treeview(frame, show="tree", selectmode="extended") + vbar = ttk.Scrollbar(frame, orient=VERTICAL, command=self.tree.yview) + self.tree.configure(yscrollcommand=vbar.set) + vbar.pack(side=RIGHT, fill=Y) + self.tree.pack(side=LEFT, fill=BOTH, expand=True) + self.tree.bind("<>", self.select_nodes) + self.tree.bind("", self.goto_node) + # The highlight is shown only while the browser has the focus. + self.bind("", self.on_focus_in) + self.bind("", self.on_focus_out) + + def configure_tag(self): + "Give the highlight tag the theme's 'hit' colors." + try: + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'hit') + except Exception: + colors = {'foreground': '#000000', 'background': '#ffff80'} + self.text.tag_configure(TAG, **colors) + + def editor_selection(self): + "Return the editor's (first, last) selection, or ('', '') if none." + try: + # A plain Text raises without a selection; the IDLE editor + # returns an empty string instead. + return self.text.index("sel.first"), self.text.index("sel.last") + except TclError: + return "", "" + + def editor_index(self, lineno, col): + "Map an AST (lineno, byte col) to an editor index, honoring the base." + if lineno <= len(self.source_lines): + # col_offset is a UTF-8 byte offset; convert it to a character one. + col = len(self.source_lines[lineno - 1].encode()[:col] + .decode(errors="replace")) + base_row, base_col = self.base + if lineno == 1: + col += base_col + return f"{base_row + lineno - 1}.{col}" + + def node_range(self, node): + "Return the (start, end) editor indices of a node, or None." + if getattr(node, "lineno", None) is None or node.end_lineno is None: + return None + return (self.editor_index(node.lineno, node.col_offset), + self.editor_index(node.end_lineno, node.end_col_offset)) + + def populate(self, event=None): + "Parse the content (or selection) and fill the tree." + self.hide_highlight() + self.tree.delete(*self.tree.get_children()) + self.ranges.clear() + text = self.text + first, last = self.editor_selection() + if first and last: + scope = "selection" + else: + last = text.index("end-1c") + # In the Shell, browse just the current input, which starts at the + # "iomark"; a plain editor has no such mark. IDLE's editor returns + # '' for a missing mark, while a plain Text raises TclError. + try: + first = text.index("iomark") + except TclError: + first = "" + if first: + scope = "input" + else: + first, scope = "1.0", "text" + self.base = tuple(int(i) for i in first.split(".")) + source = text.get(first, last) + self.source_lines = source.splitlines() + error = count = None + try: + tree = ast.parse(source) + except SyntaxError as exc: + error = exc.msg + else: + count = self.add_node("", "", tree) + status = f"{count or 0} nodes in {scope}" + if error: + status += f" — incomplete: {error}" + self.status.configure(text=status) + self.sync_from_editor() + + def add_node(self, parent_item, field, node): + "Insert a node and its descendants; return the number of nodes added." + inline = [] # Fields shown in this row: 'name=value'. + children = [] # Fields shown as child rows: (label, node). + for name, value in ast.iter_fields(node): + if isinstance(value, ast.AST): + if value._fields: + children.append((name, value)) + else: # An operator or context, e.g. Add, Load. + inline.append(f"{name}={type(value).__name__}") + elif isinstance(value, list): + nodes = [(f"{name}[{i}]", elt) for i, elt in enumerate(value) + if isinstance(elt, ast.AST)] + if nodes: + children += nodes + elif value: # A non-empty list of scalars; drop empty ones. + inline.append(f"{name}={value!r}") + elif value is not None or name == "value": # Keep the None literal. + inline.append(f"{name}={value!r}") + + label = type(node).__name__ + if inline: + label += "(" + ", ".join(inline) + ")" + if field: + label = f"{field}: {label}" + item = self.tree.insert(parent_item, END, text=label, open=True) + if rng := self.node_range(node): + self.ranges[item] = rng + + count = 1 + for name, child in children: + count += self.add_node(item, name, child) + return count + + def refresh(self): + "Re-parse the current range and bring the browser to the front." + self.populate() + self.deiconify() + self.lift() + self.focus_set() + + def sync_from_editor(self, event=None): + "Select the innermost node matching the editor's selection or cursor." + first, last = self.editor_selection() + if not (first and last): + first = last = self.text.index("insert") + self.select_rows(self.enclosing_node(first, last)) + + def enclosing_node(self, first, last): + "Return [item] of the smallest node covering [first, last], or []." + best = None + for item, (start, end) in self.ranges.items(): + if (self.text.compare(start, "<=", first) + and self.text.compare(last, "<=", end)): + # Covering nodes are nested; keep the tightest (deepest) one. + if best is None or (self.text.compare(start, ">=", best[1]) + and self.text.compare(end, "<=", best[2])): + best = (item, start, end) + return [best[0]] if best else [] + + def select_rows(self, items): + "Select the given tree rows and reveal the first." + if items: + self.tree.selection_set(items) + self.tree.focus(items[0]) + self.tree.see(items[0]) + + def select_nodes(self, event=None): + "Highlight the selected nodes and, while focused, follow with the cursor." + self.show_highlight(see=True) + # Move the editor cursor only when the browser drives the selection + # (it has the focus). When the editor drives it, the browser is not + # focused, so the cursor is left alone and there is no feedback loop. + if self.focused: + self.move_cursor() + + def show_highlight(self, see=False): + "Highlight the selected nodes' source while the browser has focus." + if not self.focused: # Keep the editor clean while it is in use. + return + text = self.text + self.hide_highlight() + first = None + for item in self.tree.selection(): + rng = self.ranges.get(item) + if rng and rng[0] != rng[1]: # Skip nodes with no source span. + text.tag_add(TAG, *rng) + if first is None: + first = rng[0] + text.tag_raise(TAG) + if see and first is not None: + text.see(first) + + def on_focus_in(self, event=None): + "Restore the highlight when the browser regains focus." + self.focused = True + self.show_highlight() + + def on_focus_out(self, event=None): + "Hide the highlight while the editor (or another window) has focus." + self.focused = False + self.hide_highlight() + + def goto_node(self, event=None): + "Move the cursor to the double-clicked node and hide the browser." + self.move_cursor(self.tree.identify_row(event.y)) + self.hide() + return "break" # Suppress the default double-click handling. + + def move_cursor(self, item=None): + "Move the editor cursor to a node (the first selected row by default)." + if item is None: + selection = self.tree.selection() + item = selection[0] if selection else None + rng = self.ranges.get(item) + if rng: + self.text.mark_set("insert", rng[0]) + self.text.see(rng[0]) + + def hide(self, event=None): + """Withdraw the browser, revealing the editor and giving it focus. + + Hiding our own window sidesteps the window manager's focus-stealing + prevention, which blocks a background editor window from being raised. + """ + self.hide_highlight() + self.withdraw() + self.text.focus_set() + + def hide_highlight(self, event=None): + try: + self.text.tag_remove(TAG, "1.0", "end") + except TclError: # The editor may already be gone. + pass + + +def _ast_browser(parent): # htest # + "Set up a sample editor Text and open an AST browser on it." + from tkinter import Text + top = Toplevel(parent) + top.title("Sample editor") + text = Text(top, width=40, height=8) + text.insert("1.0", "import sys\n\ndef f(x):\n return x + 1 # add one\n") + text.pack(fill=BOTH, expand=True) + return ASTBrowserWindow(top, text, _htest=True) + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_astbrowser', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(_ast_browser) diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py index 1fae1d4b0adbd7..e1ad4d027e2d52 100644 --- a/Lib/idlelib/debugger.py +++ b/Lib/idlelib/debugger.py @@ -170,6 +170,11 @@ def close(self, event=None): # (Causes a harmless extra cycle through close_debugger() if user # toggled debugger from pyshell Debug menu) self.pyshell.close_debugger() + # Tell observers (e.g. the disassembly browser) debugging has ended. + try: + self.pyshell.text.event_generate("<>", when="tail") + except Exception: + pass # Now close the debugger control window.... self.top.destroy() @@ -282,6 +287,9 @@ def interaction(self, message, frame, info=None): if self.vsource.get(): self.sync_source_line() + # Let observers (e.g. the disassembly browser) react to the stop. + self.pyshell.text.event_generate("<>", when="tail") + for b in self.buttons: b.configure(state="normal") @@ -313,6 +321,20 @@ def __frame2fileline(self, frame): lineno = frame.f_lineno return filename, lineno + def current_frame_code(self): + """Return (code object, last-instruction offset) for the current frame. + + The code object is the one actually executing: obtained via marshal + from a remote FrameProxy, or directly from a local frame. Return None + when the program is not stopped. + """ + frame = self.frame + if frame is None: + return None + get_code = getattr(frame, "code_object", None) # A remote FrameProxy. + code = get_code() if get_code else frame.f_code # Else a local frame. + return code, frame.f_lasti + def cont(self): self.idb.set_continue() self.abort_loop() diff --git a/Lib/idlelib/debugger_r.py b/Lib/idlelib/debugger_r.py index ad3355d9f82765..9f6c998ac5ce7f 100644 --- a/Lib/idlelib/debugger_r.py +++ b/Lib/idlelib/debugger_r.py @@ -19,6 +19,7 @@ barrier, in particular frame and traceback objects. """ +import marshal import reprlib import types from idlelib import debugger @@ -144,6 +145,12 @@ def frame_code(self, fid): codetable[cid] = code return cid + def frame_code_marshal(self, fid): + # A code object cannot be pickled through the RPC, but it can be + # marshalled; the two interpreters are the same build, so the bytes + # load back into an equivalent code object in the IDLE process. + return marshal.dumps(frametable[fid].f_code) + #----------called by a CodeProxy---------- def code_name(self, cid): @@ -219,6 +226,12 @@ def _get_f_code(self): cid = self._conn.remotecall(self._oid, "frame_code", (self._fid,), {}) return CodeProxy(self._conn, self._oid, cid) + def code_object(self): + "Return the frame's real code object, transported via marshal." + blob = self._conn.remotecall(self._oid, "frame_code_marshal", + (self._fid,), {}) + return marshal.loads(blob) + def _get_f_globals(self): did = self._conn.remotecall(self._oid, "frame_globals", (self._fid,), {}) diff --git a/Lib/idlelib/disbrowser.py b/Lib/idlelib/disbrowser.py new file mode 100644 index 00000000000000..7094badf933aea --- /dev/null +++ b/Lib/idlelib/disbrowser.py @@ -0,0 +1,518 @@ +"""A disassembly browser for IDLE. + +The Browse menu's "Disassembly Browser" command (see open() below) shows +the disassembled bytecode of the editor content, the Shell input, or the +selection, as collapsible code objects. Selecting an instruction highlights +the matching source and moves the cursor there; selecting source selects the +instructions built from it. Double-clicking a row, or Escape, hides the +browser. + +While the debugger is stopped, the browser follows it instead of the editor, +showing the code object that is executing and marking the current instruction. +""" +import dis +import os +import types + +from tkinter import Toplevel, TclError +from tkinter import TOP, BOTTOM, LEFT, RIGHT, X, Y, BOTH, W, END, VERTICAL +from tkinter import ttk + +from idlelib.config import idleConf + +# The editor tag that highlights the source of the selected instructions. +TAG = "DISBROWSER" + +# Background of the instruction the debugger is stopped at (see show_running). +CURRENT_BG = "#ffe9a8" + +# Row colors for code-object headers and for instructions grouped by the +# operand they act on (mirroring the opcode collections in the dis module). +GROUP_COLORS = { + 'code': '#7f0055', # A code-object header (BOLD MAGENTA). + 'jump': '#0000cc', # Control flow (BOLD BLUE). + 'name': '#008700', # Globals, attributes, imports (GREEN). + 'local': '#0086b3', # Locals and cells (CYAN). + 'const': '#a67c00', # Constants (YELLOW). + 'exc': '#cc0000', # Exception setup (RED). +} + + +def opcode_groups(): + "Map opcode numbers to a color group name (mirrors the dis collections)." + groups = {} + for group, opcodes in ( + ('jump', dis.hasjump), + ('name', dis.hasname), + ('local', dis.haslocal + dis.hasfree), + ('const', dis.hasconst), + ('exc', dis.hasexc)): + for op in opcodes: + groups.setdefault(op, group) + return groups + + +OPCODE_GROUPS = opcode_groups() + + +def open(editwin): + "Open the disassembly browser for editwin, reusing one already open." + window = getattr(editwin, "disassembly_browser", None) + if window is not None and window.winfo_exists(): + window.refresh() + else: + editwin.disassembly_browser = DisBrowserWindow( + editwin.top, editwin.text, editwin) + + +class DisBrowserWindow(Toplevel): + "Show the disassembly of a Text widget's content or selection." + + def __init__(self, parent, text, editwin=None, *, + _htest=False, _utest=False): + """Create the disassembly browser. + + parent - the master widget of this window. + text - the editor Text widget to browse and drive. + editwin - the owning editor window (for debugger integration), or None. + _htest - bool; change box location when running htest. + _utest - bool; don't wait for user interaction when unit testing. + """ + super().__init__(parent) + self.text = text + self.editwin = editwin + self.base = (1, 0) # Editor index of the compiled region's start. + self.source_lines = [] # Lines of the compiled source (for byte->char). + self.ranges = {} # Tree item id -> (start index, end index). + self.instr_items = set() # Tree items that are instructions (not code). + self.focused = False # Whether the browser currently has the focus. + self.debugging = False # Whether it is showing a stopped debug frame. + self.map_source = True # Whether instruction ranges map to the editor. + self.title("Disassembly Browser") + self.protocol("WM_DELETE_WINDOW", self.hide) + self.bind("", self.hide) + x = parent.winfo_rootx() + 20 + y = parent.winfo_rooty() + (100 if _htest else 20) + self.geometry(f"640x480+{x}+{y}") + self.minsize(400, 300) + + self.create_widgets() + self.configure_tag() + self.populate() + # Follow the editor and select the matching instructions. <> + # covers selection changes by keyboard or mouse (a generic + # is shadowed by IDLE's specific key bindings); the release events cover + # plain cursor moves that leave no selection. These bindings live as + # long as the editor Text and are torn down together with it (and with + # this child window), so there is nothing to unbind. + text.bind("<>", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + # Follow the debugger: it fires these on the Shell text when it stops + # at, or steps to, a frame, and when it is closed. Binding on the + # long-lived Shell text works whichever opens first. + self._shell_text = None + shell = self.shell() + if shell is not None: + self._shell_text = shell.text + self._stop_bind = shell.text.bind( + "<>", self.sync_from_debugger, add="+") + self._off_bind = shell.text.bind( + "<>", self.leave_debug_mode, add="+") + self.bind("", self.unbind_debugger) + if not _utest: + self.deiconify() + self.sync_from_debugger() # Show a frame if already stopped. + + def create_widgets(self): + bar = ttk.Frame(self, padding=(6, 6, 6, 0)) + bar.pack(side=TOP, fill=X) + ttk.Button(bar, text="Refresh", command=self.populate).pack(side=LEFT) + + self.status = ttk.Label(self, anchor=W, relief="sunken", padding=3) + self.status.pack(side=BOTTOM, fill=X) + + frame = ttk.Frame(self, padding=6) + frame.pack(side=TOP, fill=BOTH, expand=True) + # Each code object is a collapsible top-level row; the tree column + # (#0) holds its name and, for its instruction children, the offset. + self.tree = ttk.Treeview(frame, columns=("mark", "opname", "arg"), + show="tree headings", selectmode="extended") + self.tree.heading("#0", text="Code / offset", anchor=W) + self.tree.column("#0", width=170, stretch=False, anchor=W) + for name, title, width, stretch in ( + ("mark", "", 32, False), # ">>" for jump targets. + ("opname", "Instruction", 200, False), + ("arg", "Argument", 200, True)): + self.tree.heading(name, text=title, anchor=W) + self.tree.column(name, width=width, stretch=stretch, anchor=W) + for group, color in GROUP_COLORS.items(): + self.tree.tag_configure(group, foreground=color) + self.tree.tag_configure("current", background=CURRENT_BG) + vbar = ttk.Scrollbar(frame, orient=VERTICAL, command=self.tree.yview) + self.tree.configure(yscrollcommand=vbar.set) + vbar.pack(side=RIGHT, fill=Y) + self.tree.pack(side=LEFT, fill=BOTH, expand=True) + self.tree.bind("<>", self.select_instrs) + self.tree.bind("", self.goto_instr) + # The highlight is shown only while the browser has the focus. + self.bind("", self.on_focus_in) + self.bind("", self.on_focus_out) + + def configure_tag(self): + "Give the highlight tag the theme's 'hit' colors." + try: + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'hit') + except Exception: + colors = {'foreground': '#000000', 'background': '#ffff80'} + self.text.tag_configure(TAG, **colors) + + def editor_selection(self): + "Return the editor's (first, last) selection, or ('', '') if none." + try: + # A plain Text raises without a selection; the IDLE editor + # returns an empty string instead. + return self.text.index("sel.first"), self.text.index("sel.last") + except TclError: + return "", "" + + def editor_index(self, lineno, col): + "Map a (lineno, byte col) source position to an editor index." + if lineno <= len(self.source_lines): + # Position columns are UTF-8 byte offsets; convert them to chars. + col = len(self.source_lines[lineno - 1].encode()[:col] + .decode(errors="replace")) + base_row, base_col = self.base + if lineno == 1: + col += base_col + return f"{base_row + lineno - 1}.{col}" + + def clear(self): + "Empty the tree and the source-position bookkeeping." + self.hide_highlight() + self.tree.delete(*self.tree.get_children()) + self.ranges.clear() + self.instr_items.clear() + + def populate(self, event=None): + "Compile the content (or selection) and fill the tree." + self.debugging = False # Editor view: leave any debug frame behind. + self.map_source = True # Instruction positions map to the editor. + self.clear() + text = self.text + first, last = self.editor_selection() + if first and last: + scope = "selection" + else: + last = text.index("end-1c") + # In the Shell, browse just the current input, which starts at the + # "iomark"; a plain editor has no such mark. IDLE's editor returns + # '' for a missing mark, while a plain Text raises TclError. + try: + first = text.index("iomark") + except TclError: + first = "" + if first: + scope = "input" + else: + first, scope = "1.0", "text" + self.base = tuple(int(i) for i in first.split(".")) + source = text.get(first, last) + self.source_lines = source.splitlines() + error = None + try: + code = compile(source, "", "exec") + except (SyntaxError, ValueError) as exc: + error = exc.msg if isinstance(exc, SyntaxError) else str(exc) + else: + self.add_code(code) + status = f"{len(self.instr_items)} instructions in {scope}" + if error: + status += f" — incomplete: {error}" + self.status.configure(text=status) + # Every code object starts collapsed (add_code); syncing to the cursor + # opens and selects the one it is in, leaving the rest collapsed. + self.sync_from_editor() + + def add_code(self, code): + "Add a top-level code-object row with its instructions as children." + item = self.tree.insert("", END, text=code.co_qualname, + tags=("code",), open=False) + starts, ends = [], [] # Position tuples, for the code row's span. + for instr in dis.get_instructions(code): + group = OPCODE_GROUPS.get(instr.opcode) # None: the default color. + child = self.tree.insert( + item, END, text=str(instr.offset), + values=(">>" if instr.is_jump_target else "", + instr.opname, instr.argrepr), + tags=(group,) if group else ()) + pos = instr.positions + # Some instructions (e.g. a module's RESUME) carry no real source + # position: a missing field or a line before the first one. In a + # debug frame whose file is not the editor's, positions do not map + # to it, so self.map_source suppresses the (bogus) ranges. + if self.map_source and pos and None not in pos and pos.lineno >= 1: + self.ranges[child] = ( + self.editor_index(pos.lineno, pos.col_offset), + self.editor_index(pos.end_lineno, pos.end_col_offset)) + self.instr_items.add(child) + starts.append((pos.lineno, pos.col_offset)) + ends.append((pos.end_lineno, pos.end_col_offset)) + # Give the header row the span of its own instructions (compared as + # numeric positions, not editor-index strings), so selecting it + # highlights where the code lives. + if starts: + self.ranges[item] = (self.editor_index(*min(starts)), + self.editor_index(*max(ends))) + # Nested code objects (functions, classes, comprehensions, ...) follow. + for const in code.co_consts: + if isinstance(const, types.CodeType): + self.add_code(const) + + def refresh(self): + "Re-compile the current range and bring the browser to the front." + if self.debugging: + self.sync_from_debugger() # Stay on the stopped frame. + else: + self.populate() + self.deiconify() + self.lift() + self.focus_set() + + # -- Debugger integration ------------------------------------------------ + # + # While the debugger is stopped, show the frame it is stopped in: the code + # object actually executing (marshalled from the subprocess through the + # debugger RPC), with the instruction at f_lasti marked. The editor drives + # the view only when no debugger is stopped (see sync_from_editor). + + def shell(self): + "Return the Shell window that runs and debugs code, or None." + try: + return self.editwin.flist.pyshell + except AttributeError: + return None + + def active_debugger(self): + "Return the Debugger currently attached to the Shell, or None." + try: + return self.shell().interp.debugger + except AttributeError: + return None + + def unbind_debugger(self, event=None): + "Drop the Shell-text bindings when this window is destroyed." + if event is not None and event.widget is not self: + return # A child widget's , not the window's. + text, self._shell_text = self._shell_text, None + if text is not None: + try: + text.unbind("<>", self._stop_bind) + text.unbind("<>", self._off_bind) + except TclError: + pass # The Shell is already gone. + + def sync_from_debugger(self, event=None): + "Show the frame the debugger is stopped in, or leave debug mode." + debugger = self.active_debugger() + info = debugger.current_frame_code() if debugger is not None else None + if info is None: + self.leave_debug_mode() + else: + self.show_running(*info) + + def leave_debug_mode(self, event=None): + "Return from a stopped frame to the editor's own disassembly." + if self.debugging: + self.populate() # Clears self.debugging and rebuilds from editor. + + def editor_shows(self, filename): + "Whether the editor's file is the given code object's file." + try: + editor_file = self.editwin.io.filename + except AttributeError: + editor_file = None + if not editor_file or not filename: + return False + return (os.path.normcase(os.path.abspath(editor_file)) + == os.path.normcase(os.path.abspath(filename))) + + def show_running(self, code, offset): + "Disassemble a running code object; mark the instruction at offset." + self.debugging = True + self.clear() + self.base = (1, 0) + self.source_lines = [] + # Positions map to the editor only when it shows this frame's file. + self.map_source = self.editor_shows(code.co_filename) + if self.map_source: + self.source_lines = self.text.get("1.0", "end-1c").splitlines() + self.add_code(code) # The frame's own code is the first top-level row. + self.mark_current(offset) + self.status.configure( + text=f"stopped in {code.co_qualname} at offset {offset}") + + def mark_current(self, offset): + "Open, tag and reveal the instruction at offset in the frame's code." + roots = self.tree.get_children("") + if not roots: + return + code_item = roots[0] # add_code added the frame's own code first. + for child in self.tree.get_children(code_item): + if self.tree.item(child, "text") == str(offset): + # An untagged row reports its tags as '' (not an empty tuple). + tags = self.tree.item(child, "tags") + if isinstance(tags, str): + tags = (tags,) if tags else () + self.tree.item(child, tags=(*tags, "current")) + self.tree.item(code_item, open=True) + self.tree.selection_set(child) + self.tree.see(child) + break + + def editor_location(self): + "Return the (first, last) editor range to sync to (selection or cursor)." + first, last = self.editor_selection() + if not (first and last): + first = last = self.text.index("insert") + return first, last + + def code_object_at(self, first, last): + "Return the innermost top-level code row whose span covers [first, last]." + text = self.text + best = None + for item in self.tree.get_children(""): + rng = self.ranges.get(item) + if (rng and "code" in self.tree.item(item, "tags") + and text.compare(rng[0], "<=", first) + and text.compare(last, "<=", rng[1])): + # Code-object spans nest; keep the tightest (innermost) one. + if best is None or ( + text.compare(rng[0], ">=", self.ranges[best][0]) + and text.compare(rng[1], "<=", self.ranges[best][1])): + best = item + return best + + def sync_from_editor(self, event=None): + "Select the matching rows within the code object holding the location." + if self.debugging: + return # The debugger drives the view; ignore editor moves. + text = self.text + first, last = self.editor_location() + # Only the code object that the cursor/selection is in should respond; + # its own instructions, not a caller's code that merely spans the def. + target = self.code_object_at(first, last) + if target is None: + return self.select_rows([]) + rows = [it for it in self.tree.get_children(target) + if it in self.instr_items] + if first == last: # A bare cursor: rows whose range contains it. + items = [it for it in rows + if text.compare(self.ranges[it][0], "<=", first) + and text.compare(first, "<", self.ranges[it][1])] + if not items: + # At the start (indentation) or end of a line the point is in + # no half-open range; use the instructions on that line. + line = int(first.split(".")[0]) + first, last = f"{line}.0", f"{line + 1}.0" + if first != last: # A selection or a whole line: overlapping rows. + items = [it for it in rows + if text.compare(self.ranges[it][0], "<", last) + and text.compare(self.ranges[it][1], ">", first)] + if items: + self.tree.item(target, open=True) # Open it to reveal the rows. + self.select_rows(items) + + def select_rows(self, items): + "Select the given tree rows and reveal the first." + if items: + self.tree.selection_set(items) + self.tree.focus(items[0]) + self.tree.see(items[0]) + + def select_instrs(self, event=None): + "Highlight the selection and, while focused, follow with the cursor." + self.show_highlight(see=True) + # Move the editor cursor only when the browser drives the selection + # (it has the focus). When the editor drives it, the browser is not + # focused, so the cursor is left alone and there is no feedback loop. + if self.focused: + self.move_cursor() + + def show_highlight(self, see=False): + "Highlight the selected rows' source while the browser has focus." + if not self.focused: # Keep the editor clean while it is in use. + return + text = self.text + self.hide_highlight() + first = None + for item in self.tree.selection(): + rng = self.ranges.get(item) + if rng and rng[0] != rng[1]: # Skip rows with no source span. + text.tag_add(TAG, *rng) + if first is None: + first = rng[0] + text.tag_raise(TAG) + if see and first is not None: + text.see(first) + + def on_focus_in(self, event=None): + "Restore the highlight when the browser regains focus." + self.focused = True + self.show_highlight() + + def on_focus_out(self, event=None): + "Hide the highlight while the editor (or another window) has focus." + self.focused = False + self.hide_highlight() + + def goto_instr(self, event=None): + "Move the cursor to the double-clicked row and hide the browser." + self.move_cursor(self.tree.identify_row(event.y)) + self.hide() + return "break" # Suppress the default double-click handling. + + def move_cursor(self, item=None): + "Move the editor cursor to a row (the first selected row by default)." + if item is None: + selection = self.tree.selection() + item = selection[0] if selection else None + rng = self.ranges.get(item) + if rng: + self.text.mark_set("insert", rng[0]) + self.text.see(rng[0]) + + def hide(self, event=None): + """Withdraw the browser, revealing the editor and giving it focus. + + Hiding our own window sidesteps the window manager's focus-stealing + prevention, which blocks a background editor window from being raised. + """ + self.hide_highlight() + self.withdraw() + self.text.focus_set() + + def hide_highlight(self, event=None): + try: + self.text.tag_remove(TAG, "1.0", "end") + except TclError: # The editor may already be gone. + pass + + +def _disassembly_browser(parent): # htest # + "Set up a sample editor Text and open a disassembly browser on it." + from tkinter import Text + top = Toplevel(parent) + top.title("Sample editor") + text = Text(top, width=40, height=8) + text.insert("1.0", "import sys\n\ndef f(x):\n return x + 1 # add one\n") + text.pack(fill=BOTH, expand=True) + return DisBrowserWindow(top, text, _htest=True) + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_disbrowser', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(_disassembly_browser) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index a040d791bdeb52..8be687ac1473f1 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -166,6 +166,10 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<>", self.flist.close_all_callback) text.bind("<>", self.open_module_browser) text.bind("<>", self.open_path_browser) + text.bind("<>", self.open_token_browser) + text.bind("<>", self.open_ast_browser) + text.bind("<>", + self.open_disassembly_browser) text.bind("<>", self.open_turtle_demo) self.set_status_bar() @@ -425,6 +429,7 @@ def set_line_and_column(self, event=None): ("edit", "_Edit"), ("format", "F_ormat"), ("run", "_Run"), + ("browse", "_Browse"), ("options", "_Options"), ("window", "_Window"), ("help", "_Help"), @@ -740,6 +745,21 @@ def open_path_browser(self, event=None): pathbrowser.PathBrowser(self.root) return "break" + def open_token_browser(self, event=None): + from idlelib import tokenbrowser + tokenbrowser.open(self) + return "break" + + def open_ast_browser(self, event=None): + from idlelib import astbrowser + astbrowser.open(self) + return "break" + + def open_disassembly_browser(self, event=None): + from idlelib import disbrowser + disbrowser.open(self) + return "break" + def open_turtle_demo(self, event = None): import subprocess diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index 778e5c3d84e496..a3f416f721d09a 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -79,6 +79,36 @@ "Verify x.y.z versions and test each button, including Close.\n " } +_token_browser_spec = { + 'file': 'tokenbrowser', + 'kwds': {}, + 'msg': "Select rows in the token table and verify the matching regions\n" + "are highlighted in the sample editor above. Select the whole\n" + "editor text, or part of it, and press Refresh.\n" + "Double-click a row and verify the editor cursor jumps to the\n" + "start of that token and the editor gets focus." + } + +_ast_browser_spec = { + 'file': 'astbrowser', + 'kwds': {}, + 'msg': "Expand nodes in the AST tree and verify the matching source\n" + "regions are highlighted in the sample editor above. Select\n" + "text in the editor and verify the enclosing node is selected.\n" + "Double-click a node and verify the editor cursor jumps to it\n" + "and the editor gets focus." + } + +_disassembly_browser_spec = { + 'file': 'disbrowser', + 'kwds': {}, + 'msg': "Expand code objects in the tree and select instructions; verify\n" + "the matching source regions are highlighted in the sample editor\n" + "above. Select text in the editor and verify the instructions\n" + "built from it are selected. Double-click a row and verify the\n" + "editor cursor jumps to it and the editor gets focus." + } + # TODO implement ^\; adding '' to function does not work. _calltip_window_spec = { 'file': 'calltip_w', diff --git a/Lib/idlelib/idle_test/test_astbrowser.py b/Lib/idlelib/idle_test/test_astbrowser.py new file mode 100644 index 00000000000000..b073f636f1078d --- /dev/null +++ b/Lib/idlelib/idle_test/test_astbrowser.py @@ -0,0 +1,239 @@ +"Test astbrowser, coverage 95%." +from idlelib import astbrowser +from test.support import requires + +import unittest +from unittest import mock +from tkinter import Tk, Text +from idlelib.idle_test.mock_idle import Func + +code_sample = "import sys\n\ndef f(x):\n return x + 1\n" + + +def find_item(tree, label): + "Return the first tree item whose text contains label, or None." + stack = list(tree.get_children()) + while stack: + item = stack.pop(0) + if label in tree.item(item, "text"): + return item + stack[:0] = tree.get_children(item) + return None + + +class ASTBrowserOpenTest(unittest.TestCase): + "Test the open() entry point (no gui needed)." + + def make_editwin(self): + editwin = Func() # Only .top and .text are used. + editwin.top = 'toplevel' + editwin.text = 'text' + return editwin + + def test_open_creates_window(self): + editwin = self.make_editwin() + with mock.patch.object(astbrowser, 'ASTBrowserWindow', + Func(result='window')) as window: + astbrowser.open(editwin) + self.assertEqual(window.args, ('toplevel', 'text')) + self.assertEqual(editwin.ast_browser, 'window') + + def test_open_reuses_window(self): + editwin = self.make_editwin() + editwin.ast_browser = existing = Func() # A live window. + existing.winfo_exists = Func(result=1) + existing.refresh = Func() + with mock.patch.object(astbrowser, 'ASTBrowserWindow', + Func()) as new_window: + astbrowser.open(editwin) + self.assertTrue(existing.refresh.called) # Refreshed, not recreated. + self.assertFalse(new_window.called) + + +class ASTBrowserWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.window = astbrowser.ASTBrowserWindow(cls.root, cls.text, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.window.destroy() + cls.root.update_idletasks() + cls.root.destroy() + del cls.window, cls.text, cls.root + + def setUp(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", code_sample) + self.window.populate() + + def find(self, label): + "Return the first tree item whose text contains label." + item = find_item(self.window.tree, label) + if item is None: + self.fail(f"no node {label!r}") + return item + + def test_populate_text(self): + window = self.window + roots = window.tree.get_children() + self.assertEqual(len(roots), 1) + self.assertEqual(window.tree.item(roots[0], "text"), "Module") + self.assertIn("in text", window.status.cget("text")) + self.assertEqual(window.base, (1, 0)) + + def test_tree_structure(self): + # The sample parses to a tree with these nodes at some depth. + for label in ("Import", "FunctionDef(name='f')", "Return", + "BinOp(op=Add)", "Name(id='x'", "Constant(value=1)"): + self.find(label) + + def test_node_range(self): + # The Name 'x' in 'return x + 1' maps to editor coordinates. + item = self.find("Name(id='x'") + self.assertEqual(self.window.ranges[item], ("4.11", "4.12")) + + def test_selection_scope(self): + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + self.window.populate() + self.assertEqual(self.window.base, (4, 11)) + self.assertIn("in selection", self.window.status.cget("text")) + # The BinOp maps back to the selected editor coordinates. + item = self.find("BinOp") + self.assertEqual(self.window.ranges[item], ("4.11", "4.16")) + + def test_shell_input_scope(self): + text = Text(self.root) + text.insert("1.0", ">>> x = 1\n") + text.mark_set("iomark", "1.4") # After the ">>> " prompt. + window = astbrowser.ASTBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 4)) + self.assertIn("in input", window.status.cget("text")) + item = find_item(window.tree, "Name(id='x'") + self.assertEqual(window.ranges[item], ("1.4", "1.5")) + window.destroy() + text.destroy() + + def test_syntax_error(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "def f(:\n") # Invalid syntax. + self.window.populate() + self.assertIn("incomplete", self.window.status.cget("text")) + self.assertEqual(self.window.tree.get_children(), ()) + + def test_sync_cursor_selects_innermost(self): + window = self.window + self.text.mark_set("insert", "4.11") # At 'x' in 'return x + 1'. + window.sync_from_editor() + selection = window.tree.selection() + self.assertEqual(len(selection), 1) + self.assertIn("Name(id='x'", window.tree.item(selection[0], "text")) + self.assertEqual(self.text.index("insert"), "4.11") + + def test_sync_selection_selects_enclosing(self): + window = self.window + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1'. + window.sync_from_editor() + selection = window.tree.selection() + self.assertEqual(len(selection), 1) + self.assertIn("BinOp", window.tree.item(selection[0], "text")) + + def test_focused_highlights_and_moves_cursor(self): + # Browser drives the selection (it has focus): highlight the node's + # source and move the cursor to it. + window = self.window + window.focused = True + window.tree.selection_set(self.find("Name(id='x'")) + window.select_nodes() + ranges = [str(i) for i in self.text.tag_ranges(astbrowser.TAG)] + self.assertEqual(ranges, ["4.11", "4.12"]) + self.assertEqual(self.text.index("insert"), "4.11") + + def test_not_focused_keeps_editor_clean(self): + # Editor drives the selection (browser not focused): select_nodes + # neither highlights the editor nor moves its cursor. + window = self.window + window.focused = False + self.text.mark_set("insert", "1.0") + window.tree.selection_set(self.find("Name(id='x'")) + window.select_nodes() + self.assertEqual(self.text.tag_ranges(astbrowser.TAG), ()) + self.assertEqual(self.text.index("insert"), "1.0") + + def test_highlight_follows_focus(self): + window = self.window + window.tree.selection_set(self.find("Name(id='x'")) + window.on_focus_in() # The browser has focus. + self.assertNotEqual(self.text.tag_ranges(astbrowser.TAG), ()) + window.on_focus_out() # Focus moves to the editor. + self.assertEqual(self.text.tag_ranges(astbrowser.TAG), ()) + window.on_focus_in() # Focus returns to the browser. + self.assertNotEqual(self.text.tag_ranges(astbrowser.TAG), ()) + + def test_node_without_location_not_highlighted(self): + # Module has no source location: it is absent from ranges and + # selecting it produces no highlight. + window = self.window + window.focused = True + item = self.find("Module") + self.assertNotIn(item, window.ranges) + window.tree.selection_set(item) + window.select_nodes() + self.assertEqual(self.text.tag_ranges(astbrowser.TAG), ()) + + def test_refresh(self): + window = self.window + self.text.delete("1.0", "end") + self.text.insert("1.0", "spam = 1\n") + window.refresh() + self.find("Name(id='spam'") # The tree was rebuilt. + + def test_move_cursor(self): + window = self.window + item = self.find("Name(id='x'") + window.move_cursor(item) + self.assertEqual(self.text.index("insert"), "4.11") + # A node without a source range (Module) leaves the cursor put. + window.move_cursor(self.find("Module")) + self.assertEqual(self.text.index("insert"), "4.11") + + def test_hide(self): + text = Text(self.root) + text.insert("1.0", code_sample) + window = astbrowser.ASTBrowserWindow(self.root, text, _utest=True) + window.deiconify() + window.focused = True + window.tree.selection_set(find_item(window.tree, "Name(id='x'")) + window.select_nodes() + self.assertNotEqual(text.tag_ranges(astbrowser.TAG), ()) + window.hide() # Double-click (or Escape) hides it. + self.assertEqual(window.wm_state(), "withdrawn") # Not destroyed. + self.assertTrue(window.winfo_exists()) + self.assertEqual(text.tag_ranges(astbrowser.TAG), ()) + window.destroy() + text.destroy() + + def test_no_selection_empty_index(self): + # The IDLE editor returns '' (not a TclError) for a missing selection + # or mark; that must be treated as "browse the whole text", not crash. + class EditorText(Text): + def index(self, spec): + if spec.startswith("sel.") or spec == "iomark": + return "" + return super().index(spec) + text = EditorText(self.root) + text.insert("1.0", code_sample) + window = astbrowser.ASTBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 0)) + self.assertIn("in text", window.status.cget("text")) + window.destroy() + text.destroy() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index 6d75cf7aa67dcc..dd444ac27bd0be 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -424,7 +424,8 @@ def test_get_extensions(self): eq(iGE(), ['ZzDummy']) eq(iGE(editor_only=True), ['ZzDummy']) eq(iGE(active_only=False), ['ZzDummy', 'DISABLE']) - eq(iGE(active_only=False, editor_only=True), ['ZzDummy', 'DISABLE']) + eq(iGE(active_only=False, editor_only=True), + ['ZzDummy', 'DISABLE']) userextn.remove_section('ZzDummy') userextn.remove_section('DISABLE') @@ -434,7 +435,8 @@ def test_remove_key_bind_names(self): self.assertCountEqual( conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')), - ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', 'ZzDummy']) + ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', + 'ZzDummy']) def test_get_extn_name_for_event(self): userextn.read_string(''' diff --git a/Lib/idlelib/idle_test/test_debugger_r.py b/Lib/idlelib/idle_test/test_debugger_r.py index cf8af05fe27e77..00cf3b6d5964d9 100644 --- a/Lib/idlelib/idle_test/test_debugger_r.py +++ b/Lib/idlelib/idle_test/test_debugger_r.py @@ -1,6 +1,7 @@ "Test debugger_r, coverage 30%." from idlelib import debugger_r +import types import unittest # Boilerplate likely to be needed for future test classes. @@ -32,5 +33,33 @@ def __repr__(self): debugger_r.dicttable.clear() +class FrameCodeMarshalTest(unittest.TestCase): + "Transport a running frame's code object across the RPC via marshal." + + def test_round_trip(self): + code = compile("def f(x):\n return x + 1\n", "sample.py", "exec") + # Subprocess side: the adapter marshals the registered frame's code. + debugger_r.frametable[1] = types.SimpleNamespace(f_code=code) + try: + blob = debugger_r.IdbAdapter(None).frame_code_marshal(1) + finally: + debugger_r.frametable.clear() + self.assertIsInstance(blob, bytes) + + # IDLE side: a FrameProxy loads the bytes back into a code object. + class Conn: + def remotecall(self, oid, meth, args, kwargs): + self.call = (oid, meth, args) + return blob + conn = Conn() + back = debugger_r.FrameProxy(conn, 1).code_object() + self.assertEqual(conn.call, + ("idb_adapter", "frame_code_marshal", (1,))) + self.assertEqual(back.co_qualname, code.co_qualname) + self.assertEqual(back.co_code, code.co_code) + nested = [c.co_qualname for c in back.co_consts if hasattr(c, "co_code")] + self.assertIn("f", nested) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_disbrowser.py b/Lib/idlelib/idle_test/test_disbrowser.py new file mode 100644 index 00000000000000..c1801d5ebb206e --- /dev/null +++ b/Lib/idlelib/idle_test/test_disbrowser.py @@ -0,0 +1,500 @@ +"Test disbrowser, coverage 95%." +from idlelib import disbrowser +from test.support import requires + +import dis +import types +import unittest +from unittest import mock +from tkinter import Tk, Text +from idlelib.idle_test.mock_idle import Func + +code_sample = "import sys\n\ndef f(x):\n return x + 1\n" + + +def walk(tree, parent=""): + "Yield every tree item, depth first." + for item in tree.get_children(parent): + yield item + yield from walk(tree, item) + + +def find_instr(window, rng): + "The single instruction row in window whose editor range is rng." + items = [it for it in window.instr_items if window.ranges[it] == rng] + assert len(items) == 1, f"no unique instruction at {rng}" + return items[0] + + +class DisBrowserOpenTest(unittest.TestCase): + "Test the open() entry point (no gui needed)." + + def make_editwin(self): + editwin = Func() # Only .top and .text are used. + editwin.top = 'toplevel' + editwin.text = 'text' + return editwin + + def test_open_creates_window(self): + editwin = self.make_editwin() + with mock.patch.object(disbrowser, 'DisBrowserWindow', + Func(result='window')) as window: + disbrowser.open(editwin) + self.assertEqual(window.args, ('toplevel', 'text', editwin)) + self.assertEqual(editwin.disassembly_browser, 'window') + + def test_open_reuses_window(self): + editwin = self.make_editwin() + editwin.disassembly_browser = existing = Func() # A live window. + existing.winfo_exists = Func(result=1) + existing.refresh = Func() + with mock.patch.object(disbrowser, 'DisBrowserWindow', + Func()) as new_window: + disbrowser.open(editwin) + self.assertTrue(existing.refresh.called) # Refreshed, not recreated. + self.assertFalse(new_window.called) + + +class GuiTest(unittest.TestCase): + "Common Tk fixture: a hidden root and a browser on an editor Text." + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = root = Tk() + root.withdraw() + cls.text = Text(root) + cls.window = disbrowser.DisBrowserWindow(root, cls.text, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.window.destroy() + cls.root.update_idletasks() + cls.root.destroy() + del cls.window, cls.text, cls.root + + def setUp(self): + self.window.editwin = None + self.text.delete("1.0", "end") + self.text.insert("1.0", code_sample) + self.window.populate() + + +class DisBrowserWindowTest(GuiTest): + + def find_op(self, opname): + "Return the first instruction row whose Instruction column is opname." + tree = self.window.tree + for item in walk(tree): + if tree.set(item, "opname") == opname: + return item + self.fail(f"no instruction {opname!r}") + + def code_nodes(self): + "Return the top-level code-object rows, in order." + tree = self.window.tree + return [it for it in tree.get_children("") + if "code" in tree.item(it, "tags")] + + def instr_at(self, rng): + "Return the single instruction item with the given editor range." + return find_instr(self.window, rng) + + def test_populate_text(self): + window = self.window + first = window.tree.get_children("")[0] # The code row. + self.assertEqual(window.tree.item(first, "text"), "") + self.assertIn("code", window.tree.item(first, "tags")) + self.assertIn("instructions in text", window.status.cget("text")) + self.assertEqual(window.base, (1, 0)) + + def test_code_objects_at_top_level(self): + # Code objects are collapsible top-level rows (module then f), each + # with its own instruction children. + window = self.window + self.assertEqual([window.tree.item(it, "text") for it in + self.code_nodes()], ["", "f"]) + module = self.code_nodes()[0] + self.assertTrue(any(c in window.instr_items + for c in window.tree.get_children(module))) + + def test_instruction_range(self): + # The load of 'x' in 'return x + 1' maps to editor coordinates. + item = self.instr_at(("4.11", "4.12")) + self.assertEqual(self.window.tree.set(item, "arg"), "x") + + def code_node(self, qualname): + "Return the top-level code-object row named qualname." + tree = self.window.tree + for item in self.code_nodes(): + if tree.item(item, "text") == qualname: + return item + self.fail(f"no code object {qualname!r}") + + def test_code_object_span(self): + # The 'f' code row spans from its def to the end of its body. + f_node = self.code_node("f") + self.assertEqual(self.window.ranges[f_node], ("3.0", "4.16")) + + def test_open_cursor_code_object(self): + # On open, everything is collapsed and the code object holding the + # cursor is opened (the rest stay collapsed). + window = self.window + self.text.mark_set("insert", "4.11") # Inside f's body. + window.populate() + self.assertTrue(window.tree.item(self.code_node("f"), "open")) + self.assertFalse(window.tree.item(self.code_node(""), "open")) + + def test_open_module_at_module_scope(self): + window = self.window + self.text.mark_set("insert", "1.0") # On 'import sys'. + window.populate() + self.assertTrue(window.tree.item(self.code_node(""), "open")) + self.assertFalse(window.tree.item(self.code_node("f"), "open")) + + def test_all_collapsed_when_cursor_off_code(self): + # A cursor on no instruction opens nothing and selects nothing. + window = self.window + self.text.mark_set("insert", "5.0") # Past the last line. + window.populate() + self.assertFalse(any(window.tree.item(n, "open") + for n in self.code_nodes())) + self.assertEqual(window.tree.selection(), ()) + + def test_selection_scope(self): + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + self.window.populate() + self.assertEqual(self.window.base, (4, 11)) + self.assertIn("in selection", self.window.status.cget("text")) + # The load of 'x' maps back to the selected editor coordinates. + self.assertEqual(self.window.tree.set( + self.instr_at(("4.11", "4.12")), "arg"), "x") + + def test_shell_input_scope(self): + text = Text(self.root) + text.insert("1.0", ">>> x = 1\n") + text.mark_set("iomark", "1.4") # After the ">>> " prompt. + window = disbrowser.DisBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 4)) + self.assertIn("in input", window.status.cget("text")) + # The store of 'x' maps back past the ">>> " prompt to the input. + items = [it for it in window.instr_items + if window.ranges[it] == ("1.4", "1.5")] + self.assertTrue(items) + self.assertTrue(any(window.tree.set(it, "arg") == "x" + for it in items)) + window.destroy() + text.destroy() + + def test_jump_target_marked(self): + window = self.window + self.text.delete("1.0", "end") + self.text.insert("1.0", "while x:\n pass\n") # A loop has a target. + window.populate() + marks = [window.tree.set(it, "mark") for it in walk(window.tree)] + self.assertIn(">>", marks) + + def test_colorize_groups(self): + window = self.window + # Code-object rows carry the 'code' tag and its configured color. + module = self.code_node("") + self.assertIn("code", window.tree.item(module, "tags")) + self.assertEqual( + str(window.tree.tag_configure("code", "foreground")), + disbrowser.GROUP_COLORS["code"]) + # Instructions are grouped by operand kind: 'x' is a local; the + # loads/stores of names ('sys', 'f') are names; LOAD_CONST is a const. + self.assertIn("local", window.tree.item( + self.instr_at(("4.11", "4.12")), "tags")) + tags = [window.tree.item(self.find_op(a), "tags") + for a in ("STORE_NAME", "LOAD_CONST")] + self.assertIn("name", tags[0]) + self.assertIn("const", tags[1]) + + def test_colorize_jump(self): + window = self.window + self.text.delete("1.0", "end") + self.text.insert("1.0", "while x:\n pass\n") # A loop jumps. + window.populate() + self.assertTrue(any("jump" in window.tree.item(it, "tags") + for it in walk(window.tree))) + + def test_syntax_error(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "def f(:\n") # Invalid syntax. + self.window.populate() + self.assertIn("incomplete", self.window.status.cget("text")) + self.assertEqual(self.window.tree.get_children(), ()) + + def test_sync_cursor_selects_instructions(self): + window = self.window + self.text.mark_set("insert", "4.11") # At 'x' in 'return x + 1'. + window.sync_from_editor() + selection = window.tree.selection() + self.assertIn(self.instr_at(("4.11", "4.12")), selection) + self.assertEqual(self.text.index("insert"), "4.11") + + def test_sync_selection_selects_overlap(self): + window = self.window + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1'. + window.sync_from_editor() + selection = window.tree.selection() + self.assertIn(self.instr_at(("4.11", "4.12")), selection) + + def test_sync_selects_only_target_code_object(self): + # A cursor inside f selects rows in f, not the module's def-building + # instructions (which span the whole def), and opens f. + window = self.window + self.text.mark_set("insert", "4.11") # Inside f's body. + window.sync_from_editor() + selection = window.tree.selection() + f_node = self.code_node("f") + self.assertTrue(selection) + for item in selection: + self.assertEqual(window.tree.parent(item), f_node) + self.assertTrue(window.tree.item(f_node, "open")) + + def test_sync_cursor_at_line_boundaries(self): + # At the start (indentation) or end of a line, no instruction range + # contains the point, but its line's instructions are still selected. + window = self.window + f_node = self.code_node("f") + for where in ("4.0", "4.16"): # Start (indent) and end of line 4. + self.text.mark_set("insert", where) + window.sync_from_editor() + selection = window.tree.selection() + self.assertTrue(selection, f"nothing selected at {where}") + for item in selection: + self.assertEqual(window.tree.parent(item), f_node) + self.assertTrue(window.tree.item(f_node, "open")) + + def test_sync_module_level_selects_module_rows(self): + window = self.window + self.text.mark_set("insert", "1.4") # In 'import sys'. + window.sync_from_editor() + selection = window.tree.selection() + module = self.code_node("") + self.assertTrue(selection) + for item in selection: + self.assertEqual(window.tree.parent(item), module) + + def test_focused_highlights_and_moves_cursor(self): + # Browser drives the selection (it has focus): highlight the source + # and move the cursor to it. + window = self.window + window.focused = True + window.tree.selection_set(self.instr_at(("4.11", "4.12"))) + window.select_instrs() + ranges = [str(i) for i in self.text.tag_ranges(disbrowser.TAG)] + self.assertEqual(ranges, ["4.11", "4.12"]) + self.assertEqual(self.text.index("insert"), "4.11") + + def test_not_focused_keeps_editor_clean(self): + # Editor drives the selection (browser not focused): select_instrs + # neither highlights the editor nor moves its cursor. + window = self.window + window.focused = False + self.text.mark_set("insert", "1.0") + window.tree.selection_set(self.instr_at(("4.11", "4.12"))) + window.select_instrs() + self.assertEqual(self.text.tag_ranges(disbrowser.TAG), ()) + self.assertEqual(self.text.index("insert"), "1.0") + + def test_highlight_follows_focus(self): + window = self.window + window.tree.selection_set(self.instr_at(("4.11", "4.12"))) + window.on_focus_in() # The browser has focus. + self.assertNotEqual(self.text.tag_ranges(disbrowser.TAG), ()) + window.on_focus_out() # Focus moves to the editor. + self.assertEqual(self.text.tag_ranges(disbrowser.TAG), ()) + window.on_focus_in() # Focus returns to the browser. + self.assertNotEqual(self.text.tag_ranges(disbrowser.TAG), ()) + + def test_instruction_without_location_not_highlighted(self): + # The module's leading RESUME has no real source position: it is + # absent from ranges and selecting it produces no highlight. + window = self.window + window.focused = True + item = self.find_op("RESUME") # The module's, reached first. + self.assertNotIn(item, window.ranges) + window.tree.selection_set(item) + window.select_instrs() + self.assertEqual(self.text.tag_ranges(disbrowser.TAG), ()) + + def test_refresh(self): + window = self.window + self.text.delete("1.0", "end") + self.text.insert("1.0", "spam = 1\n") + window.refresh() + # The tree was rebuilt: some instruction now stores 'spam'. + self.assertTrue(any(window.tree.set(it, "arg") == "spam" + for it in window.instr_items)) + + def test_move_cursor(self): + window = self.window + window.move_cursor(self.instr_at(("4.11", "4.12"))) + self.assertEqual(self.text.index("insert"), "4.11") + # A row without a source range (the module RESUME) leaves it put. + window.move_cursor(self.find_op("RESUME")) + self.assertEqual(self.text.index("insert"), "4.11") + + def test_goto_instr(self): + # Double-clicking a row moves the cursor there and hides the browser. + text = Text(self.root) + text.insert("1.0", code_sample) + window = disbrowser.DisBrowserWindow(self.root, text, _utest=True) + item = find_instr(window, ("4.11", "4.12")) + event = Func() + event.y = 5 + with mock.patch.object(window.tree, 'identify_row', Func(result=item)): + result = window.goto_instr(event) + self.assertEqual(text.index("insert"), "4.11") + self.assertEqual(window.wm_state(), "withdrawn") + self.assertEqual(result, "break") + window.destroy() + text.destroy() + + def test_hide(self): + text = Text(self.root) + text.insert("1.0", code_sample) + window = disbrowser.DisBrowserWindow(self.root, text, _utest=True) + window.deiconify() + window.focused = True + item = find_instr(window, ("4.11", "4.12")) + window.tree.selection_set(item) + window.select_instrs() + self.assertNotEqual(text.tag_ranges(disbrowser.TAG), ()) + window.hide() # Double-click (or Escape) hides it. + self.assertEqual(window.wm_state(), "withdrawn") # Not destroyed. + self.assertTrue(window.winfo_exists()) + self.assertEqual(text.tag_ranges(disbrowser.TAG), ()) + window.destroy() + text.destroy() + + def test_no_selection_empty_index(self): + # The IDLE editor returns '' (not a TclError) for a missing selection + # or mark; that must be treated as "browse the whole text", not crash. + class EditorText(Text): + def index(self, spec): + if spec.startswith("sel.") or spec == "iomark": + return "" + return super().index(spec) + text = EditorText(self.root) + text.insert("1.0", code_sample) + window = disbrowser.DisBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 0)) + self.assertIn("in text", window.status.cget("text")) + window.destroy() + text.destroy() + + +class DisBrowserDebugTest(GuiTest): + "Test following the debugger instead of the editor (debug mode)." + + def module_code(self): + "Compile the sample as a module, with a definite filename." + return compile(code_sample, "/dir/sample.py", "exec") + + def func_code(self): + "The code object of f() nested in the sample." + return next(c for c in self.module_code().co_consts + if hasattr(c, "co_code")) + + def an_offset(self, code): + "A real instruction offset in code (past the leading RESUME)." + return list(dis.get_instructions(code))[1].offset + + def test_show_running_builds_from_code(self): + w = self.window + code = self.func_code() + w.show_running(code, self.an_offset(code)) + self.assertTrue(w.debugging) + roots = w.tree.get_children("") + self.assertEqual(w.tree.item(roots[0], "text"), code.co_qualname) + + def test_show_running_marks_current_offset(self): + w = self.window + code = self.func_code() + offset = self.an_offset(code) + w.show_running(code, offset) + selection = w.tree.selection() + self.assertEqual(len(selection), 1) + item = selection[0] + self.assertEqual(w.tree.item(item, "text"), str(offset)) + self.assertIn("current", w.tree.item(item, "tags")) + self.assertTrue(w.tree.item(w.tree.parent(item), "open")) # Revealed. + + def test_show_running_unknown_offset_marks_nothing(self): + w = self.window + code = self.func_code() + w.show_running(code, -1) # No instruction lives at offset -1. + self.assertFalse(w.tree.selection()) + + def test_leave_debug_mode_reverts_to_editor(self): + w = self.window + code = self.func_code() + w.show_running(code, self.an_offset(code)) + w.leave_debug_mode() + self.assertFalse(w.debugging) + roots = w.tree.get_children("") + self.assertEqual(w.tree.item(roots[0], "text"), "") + + def test_leave_debug_mode_when_not_debugging_is_noop(self): + w = self.window + before = w.tree.get_children("") + w.leave_debug_mode() + self.assertEqual(w.tree.get_children(""), before) + + def test_sync_from_editor_inert_while_debugging(self): + w = self.window + code = self.func_code() + w.show_running(code, self.an_offset(code)) + current = w.tree.selection() + w.text.mark_set("insert", "1.0") + w.sync_from_editor() + self.assertEqual(w.tree.selection(), current) # Unchanged. + + def test_map_source_off_when_file_differs(self): + w = self.window # editwin is None: no file to map onto. + code = self.module_code() + w.show_running(code, self.an_offset(code)) + self.assertFalse(w.map_source) + self.assertFalse(w.ranges) + + def test_map_source_on_when_file_matches(self): + w = self.window + w.editwin = types.SimpleNamespace( + io=types.SimpleNamespace(filename="/dir/sample.py")) + code = self.module_code() # Compiled with that same filename. + w.show_running(code, self.an_offset(code)) + self.assertTrue(w.map_source) + self.assertTrue(w.ranges) # Instruction positions mapped to the editor. + + def test_sync_from_debugger_shows_stopped_frame(self): + w = self.window + code = self.func_code() + offset = self.an_offset(code) + debugger = types.SimpleNamespace( + current_frame_code=lambda: (code, offset)) + w.active_debugger = lambda: debugger + try: + w.sync_from_debugger() + finally: + del w.active_debugger + self.assertTrue(w.debugging) + self.assertEqual(w.tree.item(w.tree.selection()[0], "text"), str(offset)) + + def test_sync_from_debugger_none_leaves_debug_mode(self): + w = self.window + code = self.func_code() + w.show_running(code, self.an_offset(code)) + w.active_debugger = lambda: None + try: + w.sync_from_debugger() + finally: + del w.active_debugger + self.assertFalse(w.debugging) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_mainmenu.py b/Lib/idlelib/idle_test/test_mainmenu.py index 51d2accfe48a1c..9eefe7b2e66e6e 100644 --- a/Lib/idlelib/idle_test/test_mainmenu.py +++ b/Lib/idlelib/idle_test/test_mainmenu.py @@ -11,7 +11,7 @@ class MainMenuTest(unittest.TestCase): def test_menudefs(self): actual = [item[0] for item in mainmenu.menudefs] expect = ['file', 'edit', 'format', 'run', 'shell', - 'debug', 'options', 'window', 'help'] + 'debug', 'browse', 'options', 'window', 'help'] self.assertEqual(actual, expect) def test_default_keydefs(self): diff --git a/Lib/idlelib/idle_test/test_tokenbrowser.py b/Lib/idlelib/idle_test/test_tokenbrowser.py new file mode 100644 index 00000000000000..bdded543e0f122 --- /dev/null +++ b/Lib/idlelib/idle_test/test_tokenbrowser.py @@ -0,0 +1,286 @@ +"Test tokenbrowser, coverage 95%." +from idlelib import tokenbrowser +from test.support import requires + +import unittest +from unittest import mock +from tkinter import Tk, Text +from idlelib.idle_test.mock_idle import Func + +code_sample = "import sys\n\ndef f(x):\n return x + 1\n" + + +class TokenBrowserOpenTest(unittest.TestCase): + "Test the open() entry point (no gui needed)." + + def make_editwin(self): + editwin = Func() # Only .top and .text are used. + editwin.top = 'toplevel' + editwin.text = 'text' + return editwin + + def test_open_creates_window(self): + editwin = self.make_editwin() + with mock.patch.object(tokenbrowser, 'TokenBrowserWindow', + Func(result='window')) as window: + tokenbrowser.open(editwin) + self.assertEqual(window.args, ('toplevel', 'text')) + self.assertEqual(editwin.token_browser, 'window') + + def test_open_reuses_window(self): + editwin = self.make_editwin() + editwin.token_browser = existing = Func() # A live window. + existing.winfo_exists = Func(result=1) + existing.refresh = Func() + with mock.patch.object(tokenbrowser, 'TokenBrowserWindow', + Func()) as new_window: + tokenbrowser.open(editwin) + self.assertTrue(existing.refresh.called) # Refreshed, not recreated. + self.assertFalse(new_window.called) + + +class TokenBrowserWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.window = tokenbrowser.TokenBrowserWindow( + cls.root, cls.text, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.window.destroy() + cls.root.update_idletasks() + cls.root.destroy() + del cls.window, cls.text, cls.root + + def setUp(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", code_sample) + self.window.populate() + + def find(self, type=None, string=None): + "Return the first tree item matching a token type and/or string." + tree = self.window.tree + for item in tree.get_children(): + typ, s = tree.item(item, "values") + if (type is None or typ == type) and \ + (string is None or s == repr(string)): + return item + self.fail(f"no token {type} {string!r}") + + def test_populate_text(self): + window = self.window + self.assertGreater(len(window.ranges), 0) + self.assertEqual(len(window.ranges), len(window.tree.get_children())) + self.assertIn("in text", window.status.cget("text")) + self.assertEqual(window.base, (1, 0)) + + def test_token_row_values(self): + tree = self.window.tree + item = tree.get_children()[0] + # First token is NAME 'import', shown as two columns, mapped to 1.0-1.6. + self.assertEqual(tree.item(item, "values"), ("NAME", repr("import"))) + self.assertEqual(self.window.ranges[item], ("1.0", "1.6")) + # Operators show their exact type. + self.find(type="PLUS", string="+") + + def test_token_colors(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "x = 'a' + 1 # c\n") + self.window.populate() + tree = self.window.tree + tags = {tree.item(item, "values")[1]: tree.item(item, "tags") + for item in tree.get_children()} + self.assertIn("string", tags[repr("'a'")]) + self.assertIn("number", tags[repr("1")]) + self.assertIn("comment", tags[repr("# c")]) + self.assertNotIn("string", tags[repr("x")]) # NAME: default color. + + def test_editor_index(self): + window = self.window + window.base = (1, 0) + self.assertEqual(window.editor_index(1, 0), "1.0") + self.assertEqual(window.editor_index(3, 4), "3.4") + + def test_selection_scope(self): + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + self.window.populate() + window = self.window + self.assertEqual(window.base, (4, 11)) + self.assertIn("in selection", window.status.cget("text")) + # Tokens map back to editor coordinates. + item = self.find(type="NAME", string="x") + self.assertEqual(window.ranges[item], ("4.11", "4.12")) + + def test_focused_highlights_and_moves_cursor(self): + # Browser drives the selection (it has focus): highlight the token + # in the editor and move the cursor to it. + window = self.window + window.focused = True + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.select_tokens() + ranges = [str(i) for i in self.text.tag_ranges(tokenbrowser.TAG)] + self.assertEqual(ranges, ["1.7", "1.10"]) + self.assertEqual(self.text.index("insert"), "1.7") + + def test_not_focused_keeps_editor_clean(self): + # Editor drives the selection (browser not focused): select_tokens + # neither highlights the editor nor moves its cursor. + window = self.window + window.focused = False + self.text.mark_set("insert", "1.0") + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.select_tokens() + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + self.assertEqual(self.text.index("insert"), "1.0") + + def test_select_multiple_highlights(self): + window = self.window + window.focused = True + items = [self.find(type="NAME", string="import"), + self.find(type="NAME", string="sys")] + window.tree.selection_set(items) + window.select_tokens() + ranges = self.text.tag_ranges(tokenbrowser.TAG) + self.assertEqual(len(ranges), 4) # Two (start, end) pairs. + + def test_highlight_follows_focus(self): + window = self.window + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.on_focus_in() # The browser has focus. + self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + window.on_focus_out() # Focus moves to the editor. + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + window.on_focus_in() # Focus returns to the browser. + self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + + def test_extend_selection(self): + tree = self.window.tree + rows = tree.get_children() + tree.selection_set(rows[0]) + tree.focus(rows[0]) + self.window.extend_selection(1) + self.assertEqual(set(tree.selection()), {rows[0], rows[1]}) + self.window.extend_selection(1) + self.assertEqual(set(tree.selection()), {rows[0], rows[1], rows[2]}) + + def test_extend_selection_at_edge(self): + tree = self.window.tree + last = tree.get_children()[-1] + tree.selection_set(last) + tree.focus(last) + self.window.extend_selection(1) # No next row to add. + self.assertEqual(tree.selection(), (last,)) + + def test_zero_width_not_highlighted(self): + window = self.window + window.focused = True + item = self.find(type="ENDMARKER") + start, end = window.ranges[item] + self.assertEqual(start, end) + window.tree.selection_set(item) + window.select_tokens() + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + + def test_sync_cursor_row(self): + # With no editor selection, sync selects the single row of the + # token under the cursor, without moving the cursor. + window = self.window + self.text.mark_set("insert", "1.8") # Inside 'sys' (1.7-1.10). + window.sync_from_editor() + selection = window.tree.selection() + self.assertEqual(len(selection), 1) + self.assertEqual(window.tree.item(selection[0], "values"), + ("NAME", repr("sys"))) + self.assertEqual(self.text.index("insert"), "1.8") + + def test_sync_selection_selects_rows(self): + # An editor selection selects every overlapping token's row. + window = self.window + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + window.sync_from_editor() + values = {window.tree.item(item, "values") + for item in window.tree.selection()} + self.assertEqual(values, {("NAME", repr("x")), + ("PLUS", repr("+")), + ("NUMBER", repr("1"))}) + + def test_refresh(self): + window = self.window + self.text.delete("1.0", "end") + self.text.insert("1.0", "spam = 1\n") + window.refresh() + strings = [window.tree.item(i, "values")[1] + for i in window.tree.get_children()] + self.assertIn(repr("spam"), strings) + + def test_move_cursor(self): + window = self.window + item = self.find(type="NAME", string="return") + window.move_cursor(item) + self.assertEqual(self.text.index("insert"), window.ranges[item][0]) + + def test_move_cursor_no_item(self): + self.window.move_cursor("") # identify_row returns "" off a row. + + def test_hide(self): + text = Text(self.root) + text.insert("1.0", code_sample) + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + window.deiconify() + window.focused = True + window.tree.selection_set(window.tree.get_children()[0]) + window.select_tokens() + self.assertNotEqual(text.tag_ranges(tokenbrowser.TAG), ()) + window.hide() # Double-click (or Escape) hides it. + self.assertEqual(window.wm_state(), "withdrawn") # Not destroyed. + self.assertTrue(window.winfo_exists()) + self.assertEqual(text.tag_ranges(tokenbrowser.TAG), ()) + window.destroy() + text.destroy() + + def test_shell_input_scope(self): + # In the Shell (a Text with an "iomark"), browse only the current + # input, which starts after the prompt at the iomark. + text = Text(self.root) + text.insert("1.0", ">>> x = 1\n") + text.mark_set("iomark", "1.4") # After the ">>> " prompt. + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 4)) + self.assertIn("in input", window.status.cget("text")) + # The prompt is not tokenized; the first token is NAME 'x' at 1.4. + first = window.tree.get_children()[0] + self.assertEqual(window.tree.item(first, "values"), ("NAME", repr("x"))) + self.assertEqual(window.ranges[first], ("1.4", "1.5")) + window.destroy() + text.destroy() + + def test_no_selection_empty_index(self): + # The IDLE editor returns '' (not a TclError) for a missing selection + # or mark; that must be treated as "browse the whole text", not crash. + class EditorText(Text): + def index(self, spec): + if spec.startswith("sel.") or spec == "iomark": + return "" + return super().index(spec) + text = EditorText(self.root) + text.insert("1.0", code_sample) + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 0)) + self.assertIn("in text", window.status.cget("text")) + window.destroy() + text.destroy() + + def test_incomplete_source(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "def f(:\n") # Unbalanced/invalid. + self.window.populate() + self.assertIn("incomplete", self.window.status.cget("text")) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/mainmenu.py b/Lib/idlelib/mainmenu.py index 91a32cebb513f9..92c9d91b94fc76 100644 --- a/Lib/idlelib/mainmenu.py +++ b/Lib/idlelib/mainmenu.py @@ -97,6 +97,12 @@ ('!_Auto-open Stack Viewer', '<>'), ]), + ('browse', [ + ('_Token Browser', '<>'), + ('_AST Browser', '<>'), + ('_Disassembly Browser', '<>'), + ]), + ('options', [ ('Configure _IDLE', '<>'), None, diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index ef3d014d936ce8..1b59807ff5277f 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -865,6 +865,7 @@ class PyShell(OutputWindow): ("file", "_File"), ("edit", "_Edit"), ("debug", "_Debug"), + ("browse", "_Browse"), ("options", "_Options"), ("window", "_Window"), ("help", "_Help"), diff --git a/Lib/idlelib/tokenbrowser.py b/Lib/idlelib/tokenbrowser.py new file mode 100644 index 00000000000000..a6d41680d68009 --- /dev/null +++ b/Lib/idlelib/tokenbrowser.py @@ -0,0 +1,345 @@ +"""A token browser for IDLE. + +The Browse menu's "Token Browser" command (see open() below) opens a +window listing the Python tokens of the editor content (or, in the Shell, +the current input), or of the selection if there is one. Selecting rows +highlights the matching regions in the editor and moves the editor cursor +there; selecting text (or moving the cursor) in the editor selects the +matching rows. Double-clicking a row hides the browser (as does Escape), +revealing the editor at the token. +""" +import io +import token +import tokenize + +from tkinter import Toplevel, TclError +from tkinter import TOP, BOTTOM, LEFT, RIGHT, X, Y, BOTH, W, END, VERTICAL +from tkinter import ttk + +from idlelib.config import idleConf + +# The editor tag that highlights the tokens of the selected rows. +TAG = "TOKENBROWSER" + +# Row colors per token group, mirroring the "python -m tokenize" CLI +# (see tokenize._get_token_colors and _colorize.Syntax/Tokenize). Token +# groups not listed here (NAME, OP) keep the default foreground. +GROUP_COLORS = { + 'comment': '#cc0000', # RED + 'string': '#008700', # GREEN + 'number': '#a67c00', # YELLOW + 'soft_keyword': '#0000cc', # BOLD_BLUE + 'whitespace': '#808080', # GREY + 'error': '#e40000', # BOLD_RED +} + + +def token_groups(): + "Map token type numbers to a color group name (mirrors the CLI)." + groups = {} + for group, names in ( + ('comment', ['COMMENT']), + ('whitespace', ['DEDENT', 'ENCODING', 'ENDMARKER', 'INDENT', + 'NEWLINE', 'NL']), + ('error', ['ERRORTOKEN']), + ('string', ['STRING', 'FSTRING_START', 'FSTRING_MIDDLE', + 'FSTRING_END', 'TSTRING_START', 'TSTRING_MIDDLE', + 'TSTRING_END']), + ('number', ['NUMBER']), + ('soft_keyword', ['SOFT_KEYWORD'])): + for name in names: + value = getattr(token, name, None) + if value is not None: # Some token types are version-specific. + groups[value] = group + return groups + + +TOKEN_GROUPS = token_groups() + + +def open(editwin): + "Open the token browser for editwin, reusing one already open." + window = getattr(editwin, "token_browser", None) + if window is not None and window.winfo_exists(): + window.refresh() + else: + editwin.token_browser = TokenBrowserWindow(editwin.top, editwin.text) + + +class TokenBrowserWindow(Toplevel): + "List the Python tokens of a Text widget's content or selection." + + def __init__(self, parent, text, *, _htest=False, _utest=False): + """Create the token browser. + + parent - the master widget of this window. + text - the editor Text widget to browse and drive. + _htest - bool; change box location when running htest. + _utest - bool; don't wait for user interaction when unit testing. + """ + super().__init__(parent) + self.text = text + self.base = (1, 0) # Editor index of the tokenized region's start. + self.ranges = {} # Tree item id -> (start index, end index). + self.focused = False # Whether the browser currently has the focus. + self.title("Token Browser") + self.protocol("WM_DELETE_WINDOW", self.hide) + self.bind("", self.hide) + x = parent.winfo_rootx() + 20 + y = parent.winfo_rooty() + (100 if _htest else 20) + self.geometry(f"640x480+{x}+{y}") + self.minsize(400, 300) + + self.create_widgets() + self.configure_tag() + self.populate() + # Follow the editor and select the matching rows. <> + # covers selection changes by keyboard or mouse (a generic + # is shadowed by IDLE's specific key bindings); the release events + # cover plain cursor moves that leave no selection. These bindings + # live as long as the editor Text and are torn down together with it + # (and with this child window), so there is nothing to unbind. + text.bind("<>", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + if not _utest: + self.deiconify() + + def create_widgets(self): + bar = ttk.Frame(self, padding=(6, 6, 6, 0)) + bar.pack(side=TOP, fill=X) + ttk.Button(bar, text="Refresh", command=self.populate).pack(side=LEFT) + + self.status = ttk.Label(self, anchor=W, relief="sunken", padding=3) + self.status.pack(side=BOTTOM, fill=X) + + frame = ttk.Frame(self, padding=6) + frame.pack(side=TOP, fill=BOTH, expand=True) + self.tree = ttk.Treeview(frame, columns=("type", "string"), + show="headings", selectmode="extended") + for name, title, width, stretch in ( + ("type", "Type", 120, False), + ("string", "String", 260, True)): + self.tree.heading(name, text=title) + self.tree.column(name, width=width, stretch=stretch, anchor=W) + for group, color in GROUP_COLORS.items(): + self.tree.tag_configure(group, foreground=color) + vbar = ttk.Scrollbar(frame, orient=VERTICAL, command=self.tree.yview) + self.tree.configure(yscrollcommand=vbar.set) + vbar.pack(side=RIGHT, fill=Y) + self.tree.pack(side=LEFT, fill=BOTH, expand=True) + self.tree.bind("<>", self.select_tokens) + self.tree.bind("", self.goto_token) + # Shift + Up/Down extends the selection with the keyboard. + self.tree.bind("", lambda e: self.extend_selection(-1)) + self.tree.bind("", lambda e: self.extend_selection(1)) + # The highlight is shown only while the browser has the focus. + self.bind("", self.on_focus_in) + self.bind("", self.on_focus_out) + + def configure_tag(self): + "Give the highlight tag the theme's 'hit' colors." + try: + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'hit') + except Exception: + colors = {'foreground': '#000000', 'background': '#ffff80'} + self.text.tag_configure(TAG, **colors) + + def editor_index(self, row, col): + "Map a token (row, col) to an editor index, honoring the selection." + base_row, base_col = self.base + if row == 1: + col += base_col + return f"{base_row + row - 1}.{col}" + + def editor_selection(self): + "Return the editor's (first, last) selection, or ('', '') if none." + try: + # A plain Text raises without a selection; the IDLE editor + # returns an empty string instead. + return self.text.index("sel.first"), self.text.index("sel.last") + except TclError: + return "", "" + + def populate(self, event=None): + "Tokenize the content (or selection) and fill the table." + self.hide_highlight() + self.tree.delete(*self.tree.get_children()) + self.ranges.clear() + text = self.text + first, last = self.editor_selection() + if first and last: + scope = "selection" + else: + last = text.index("end-1c") + # In the Shell, browse just the current input, which starts at the + # "iomark"; a plain editor has no such mark. IDLE's editor returns + # '' for a missing mark, while a plain Text raises TclError. + try: + first = text.index("iomark") + except TclError: + first = "" + if first: + scope = "input" + else: + first, scope = "1.0", "text" + self.base = tuple(int(i) for i in first.split(".")) + source = text.get(first, last) + if not source.endswith("\n"): + source += "\n" + error = None + try: + for tok in tokenize.generate_tokens(io.StringIO(source).readline): + self.add_token(tok) + except (tokenize.TokenError, IndentationError, SyntaxError) as exc: + error = exc.args[0] if exc.args else type(exc).__name__ + status = f"{len(self.ranges)} tokens in {scope}" + if error: + status += f" — incomplete: {error}" + self.status.configure(text=status) + self.sync_from_editor() + + def refresh(self): + "Re-tokenize the current range and bring the browser to the front." + self.populate() + self.deiconify() + self.lift() + self.focus_set() + + def sync_from_editor(self, event=None): + "Select the rows matching the editor's selection, or the cursor's row." + first, last = self.editor_selection() + if first and last: + # Select every token whose range overlaps the editor selection. + text = self.text + self.select_rows( + [item for item, (start, end) in self.ranges.items() + if text.compare(start, "<", last) + and text.compare(end, ">", first)]) + else: + self.select_cursor_row() + + def select_cursor_row(self): + "Select the row of the token that contains the editor's cursor." + insert = self.text.index("insert") + chosen = None + for item, (start, end) in self.ranges.items(): + if self.text.compare(start, "<=", insert): + chosen = item # Last token starting at or before it. + if self.text.compare(insert, "<", end): + break # The cursor is inside this token. + self.select_rows([chosen] if chosen else []) + + def select_rows(self, items): + "Select the given tree rows and reveal the first." + if items: + self.tree.selection_set(items) + self.tree.focus(items[0]) + self.tree.see(items[0]) + + def add_token(self, tok): + name = token.tok_name[tok.exact_type] + start = self.editor_index(*tok.start) + end = self.editor_index(*tok.end) + group = TOKEN_GROUPS.get(tok.type, '') # '' means the default color. + item = self.tree.insert("", END, values=(name, repr(tok.string)), + tags=(group,) if group else ()) + self.ranges[item] = (start, end) + + def select_tokens(self, event=None): + "Highlight the selected rows and, while focused, follow with the cursor." + self.show_highlight(see=True) + # Move the editor cursor only when the browser drives the selection + # (it has the focus). When the editor drives it, the browser is not + # focused, so the cursor is left alone and there is no feedback loop. + if self.focused: + self.move_cursor() + + def show_highlight(self, see=False): + "Highlight the selected rows' tokens while the browser has focus." + if not self.focused: # Keep the editor clean while it is in use. + return + text = self.text + self.hide_highlight() + first = None + for item in self.tree.selection(): + start, end = self.ranges[item] + if start != end: # Skip zero-width tokens (NEWLINE, INDENT, ...). + text.tag_add(TAG, start, end) + if first is None: + first = start + text.tag_raise(TAG) + if see and first is not None: + text.see(first) + + def on_focus_in(self, event=None): + "Restore the highlight when the browser regains focus." + self.focused = True + self.show_highlight() + + def on_focus_out(self, event=None): + "Hide the highlight while the editor (or another window) has focus." + self.focused = False + self.hide_highlight() + + def extend_selection(self, direction): + "Extend the selection to the previous or next row (Shift+Up/Down)." + tree = self.tree + item = tree.next(tree.focus()) if direction > 0 else tree.prev(tree.focus()) + if item: + tree.selection_add(item) + tree.focus(item) + tree.see(item) + return "break" + + def goto_token(self, event=None): + "Move the cursor to the double-clicked token and hide the browser." + self.move_cursor(self.tree.identify_row(event.y)) + self.hide() + return "break" # Suppress the default double-click handling. + + def move_cursor(self, item=None): + "Move the editor cursor to a token (the first selected row by default)." + if item is None: + selection = self.tree.selection() + item = selection[0] if selection else None + if not item: + return + start, end = self.ranges[item] + self.text.mark_set("insert", start) + self.text.see(start) + + def hide(self, event=None): + """Withdraw the browser, revealing the editor and giving it focus. + + Hiding our own window sidesteps the window manager's focus-stealing + prevention, which blocks a background editor window from being raised. + """ + self.hide_highlight() + self.withdraw() + self.text.focus_set() + + def hide_highlight(self, event=None): + try: + self.text.tag_remove(TAG, "1.0", "end") + except TclError: # The editor may already be gone. + pass + + +def _token_browser(parent): # htest # + "Set up a sample editor Text and open a token browser on it." + from tkinter import Text + top = Toplevel(parent) + top.title("Sample editor") + text = Text(top, width=40, height=8) + text.insert("1.0", "import sys\n\ndef f(x):\n return x + 1 # add one\n") + text.pack(fill=BOTH, expand=True) + return TokenBrowserWindow(top, text, _htest=True) + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_tokenbrowser', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(_token_browser) diff --git a/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst b/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst new file mode 100644 index 00000000000000..0cf33bcdfea92e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst @@ -0,0 +1,3 @@ +Add a Token Browser to IDLE, opened from the new Browse menu. It lists the +Python tokens of the editor content, the Shell input, or the selection, with +token type names colored as by ``python -m tokenize``. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-03-12-30-00.gh-issue-152942.AsTbRo.rst b/Misc/NEWS.d/next/IDLE/2026-07-03-12-30-00.gh-issue-152942.AsTbRo.rst new file mode 100644 index 00000000000000..a8ef25ef838e74 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-03-12-30-00.gh-issue-152942.AsTbRo.rst @@ -0,0 +1,4 @@ +Add an AST Browser to IDLE, opened from the Browse menu. It shows the abstract +syntax tree of the editor content, the Shell input, or the selection. +Selecting a node highlights the matching region in the editor, and selecting +text in the editor selects the innermost enclosing node. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-03-13-00-00.gh-issue-152966.DisBro.rst b/Misc/NEWS.d/next/IDLE/2026-07-03-13-00-00.gh-issue-152966.DisBro.rst new file mode 100644 index 00000000000000..f830ea3200f6fa --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-03-13-00-00.gh-issue-152966.DisBro.rst @@ -0,0 +1,6 @@ +Add a Disassembly Browser to IDLE, opened from the Browse menu. It shows the +disassembled bytecode of the editor content, the Shell input, or the +selection, as collapsible code objects colored by operand kind. Selecting an +instruction highlights the matching source, and selecting source selects the +instructions. While the debugger is stopped, the browser instead shows the +code object that is executing and marks the current instruction.