gh-159465: Improve incomplete input detection

Author: pablogsal

Include/errcode.h

@@ -38,6 +38,7 @@ extern "C" {
 #define E_BADSINGLE      27      /* Ill-formed single statement input */
 #define E_INTERACT_STOP  28      /* Interactive mode stopped tokenization */
 #define E_COLUMNOVERFLOW 29      /* Column offset overflow */
+#define E_INCOMPLETE     30      /* An unfinished token in partial input */
 
 #ifdef __cplusplus
 }

Include/internal/pycore_parser.h

@@ -27,13 +27,17 @@ _Py_DECLARE_STR(empty, "")
         }, \
     }
 
+/* If incomplete is non-NULL, initialize *incomplete to NULL. A returned AST
+   can also return an owned exception there for a missing line terminator.
+   Validate the AST before reporting that exception. */
 extern struct _mod* _PyParser_ASTFromString(
     const char *str,
     PyObject* filename,
     int mode,
     PyCompilerFlags *flags,
     PyArena *arena,
-    PyObject *module);
+    PyObject *module,
+    PyObject **incomplete);
 
 extern struct _mod* _PyParser_ASTFromFile(
     FILE *fp,

Lib/codeop.py

@@ -62,15 +62,10 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
         warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
         try:
             compiler(source, filename, symbol, flags=flags)
+        except _IncompleteInputError:
+            return None
         except SyntaxError:  # Let other compile() errors propagate.
-            try:
-                compiler(source + "\n", filename, symbol, flags=flags)
-                return None
-            except _IncompleteInputError:
-                return None
-            except SyntaxError:
-                pass
-                # fallthrough
+            pass
 
     return compiler(source, filename, symbol, incomplete_input=False)
 

Lib/test/test_codeop.py

@@ -124,6 +124,71 @@ def test_incomplete_string_diagnostics(self, symbol, prefix):
         self.assertEqual(cm.exception.args, (
             'incomplete input', ('<input>', 2, 9, text, 2, -1)))
 
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    def test_unfinished_token_in_invalid_position(self, symbol):
+        for source in ('a b', '(a b', '1 """', '(1 """',
+                       'f"{lambda:\\', 't"{lambda:\\'):
+            for compiler in COMPILERS:
+                with self.subTest(source=source, compiler=compiler):
+                    with self.assertRaises(SyntaxError) as cm:
+                        compiler(source, '<input>', symbol)
+                    self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    def test_invalid_partial_string_diagnostic(self, symbol):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            Compile()('a"""', '<input>', symbol)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('', 'r', 'b', 'f', 't', 'rf', 'rt'))
+    def test_string_continuation(self, symbol, prefix):
+        source = prefix + '"first\\'
+        compile(source + '\nsecond"\n', '<input>', symbol)
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('newline', ('\n', '\r', '\r\n'))
+    def test_continued_final_line(self, symbol, newline):
+        source = '1\\' + newline
+        compile(source + '+ 2\n', '<input>', symbol)
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+        with self.assertRaises(SyntaxError) as cm:
+            Compile()('1 + \\' + newline, '<input>', symbol)
+        # Appending LF to CR completes the CRLF line terminator.
+        if newline != '\r':
+            self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+
+    @subTests('symbol', ('single', 'exec'))
+    def test_eof_before_nested_body(self, symbol):
+        for source in ('if True:\n    if True:',
+                       'if True:\n    def f():',
+                       'match 1:\n    case 1:'):
+            for tail in ('', '\n', '\n\n'):
+                with self.subTest(source=source, tail=tail):
+                    compile(source + tail + '\n        pass\n', '<input>', symbol)
+                    self.assertIsNone(compile_command(source + tail, symbol=symbol))
+                    with self.assertRaises(_IncompleteInputError):
+                        Compile()(source + tail, '<input>', symbol)
+
+    def test_incomplete_suite_semantic_errors(self):
+        cases = (
+            ('if True: break', "'break' outside loop"),
+            ('if True: return 1', "'return' outside function"),
+            ('def f(x, x):\n    pass', 'duplicate parameter'),
+        )
+        for source, message in cases:
+            for compiler in COMPILERS:
+                with self.subTest(source=source, compiler=compiler):
+                    with self.assertRaises(SyntaxError) as cm:
+                        compiler(source, '<input>', 'single')
+                    self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+                    if compiler in RAW_COMPILERS:
+                        self.assertIn(message, str(cm.exception))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst

@@ -0,0 +1,2 @@
+Improve incomplete input detection in :mod:`codeop` without an extra parser
+pass, and preserve syntax and compiler errors.

Modules/_testinternalcapi/tokenizer.c

@@ -95,6 +95,35 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
+    _PyTok_SourceClear(&source);
+    if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) == 0,
+              "cannot append empty terminated line") < 0 ||
+            check(_PyTok_SourceAppendLineWithNewline(&source, tail, 4) == 1,
+                  "wrong terminated line offset") < 0 ||
+            check(source.len == 6 && source.nlines == 2 &&
+                      memcmp(source.bytes, "\ntail\n", 7) == 0,
+                  "wrong terminated source contents") < 0 ||
+            check_system_error(_PyTok_SourceAppendLineWithNewline(
+                &source, terminated_line, 2) < 0,
+                "accepted newline in line payload") < 0) {
+        goto error;
+    }
+    source.base_offset = PY_SSIZE_T_MAX - source.len;
+    if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 &&
+                  PyErr_ExceptionMatches(PyExc_MemoryError),
+              "accepted overflowing appended newline") < 0) {
+        goto error;
+    }
+    PyErr_Clear();
+    source.base_offset = 0;
+    source.nlines = INT_MAX - 1;
+    if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 &&
+                  PyErr_ExceptionMatches(PyExc_OverflowError),
+              "accepted overflowing newline count") < 0) {
+        goto error;
+    }
+    PyErr_Clear();
+
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;
 

Parser/action_helpers.c

@@ -1497,11 +1497,11 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b
 }
 
 expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
-    Py_ssize_t bsize;
-    char* bstr;
-    if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) {
-        return NULL;
+    if (tok->bytes == NULL) {
+        return _PyPegen_raise_incomplete_input(p);
     }
+    Py_ssize_t bsize = PyBytes_GET_SIZE(tok->bytes);
+    const char *bstr = PyBytes_AS_STRING(tok->bytes);
 
     int is_raw = tok->is_raw;
 
@@ -1519,10 +1519,10 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
 }
 
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
-    const char* bstr = PyBytes_AsString(tok->bytes);
-    if (bstr == NULL) {
-        return NULL;
+    if (tok->bytes == NULL) {
+        return _PyPegen_raise_incomplete_input(p);
     }
+    const char *bstr = PyBytes_AS_STRING(tok->bytes);
     PyObject* str = PyUnicode_FromString(bstr);
     if (str == NULL) {
         return NULL;
@@ -1537,10 +1537,10 @@ expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
 }
 
 expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok) {
-    const char* the_str = PyBytes_AsString(tok->bytes);
-    if (the_str == NULL) {
-        return NULL;
+    if (tok->bytes == NULL) {
+        return _PyPegen_raise_incomplete_input(p);
     }
+    const char *the_str = PyBytes_AS_STRING(tok->bytes);
     PyObject *s = _PyPegen_parse_string(p, tok);
     if (s == NULL) {
         _Pypegen_raise_decode_error(p);

Parser/lexer/layout.c

@@ -21,6 +21,9 @@ _PyLexer_ContinueLine(struct tok_state *tok)
         return -1;
     }
     c = tok_nextc(tok);
+    if (tok->implicit_newline == _PyTokenizer_APPEND_NEWLINE) {
+        tok->needs_newline = 1;
+    }
     if (c == EOF) {
         tok->done = E_EOF;
         tok->cur = tok->inp;

Parser/lexer/lexer.c

@@ -516,7 +516,7 @@ _PyTokenizer_Get(struct tok_state *tok, struct token *token)
             Py_UNREACHABLE();
         }
     }
-    if (tok_failed(tok)) {
+    if (tok_failed(tok) && tok->done != E_INCOMPLETE) {
         result = ERRORTOKEN;
     }
     token->type = result;

Parser/lexer/state.h

@@ -105,6 +105,8 @@ struct tok_state {
     int debug;
 #endif
     int barry_as_bdfl;
+    unsigned char allow_incomplete_input;
+    unsigned char needs_newline;
 };
 
 static inline ftstring_state *