gh-159465: Track newline effects and preserve parser layout

Author: pablogsal

Lib/test/test_codeop.py

@@ -124,6 +124,34 @@ def test_incomplete_string_diagnostics(self, symbol, prefix):
         self.assertEqual(cm.exception.args, (
             'incomplete input', ('<input>', 2, 9, text, 2, -1)))
 
+    @subTests('compiler', COMPILERS)
+    @subTests('newline', ('\n', '\r', '\r\n'))
+    def test_unterminated_trailing_line(self, compiler, newline):
+        for head, symbol in (('1', 'eval'), ('if 1: pass', 'single'),
+                             ('if 1:\n pass', 'single')):
+            for tail in (' ', '\t', '\f ', '\n '):
+                source = head + newline + tail
+                with self.subTest(source=source, symbol=symbol):
+                    if compiler in RAW_COMPILERS:
+                        with self.assertRaises(_IncompleteInputError):
+                            compiler(source, '<input>', symbol)
+                    else:
+                        self.assertIsNone(compiler(source, '<input>', symbol))
+                    self.assertValid(source + '\n', symbol, compiler=compiler)
+            for tail in ('\f', ' \f', '\n\f'):
+                with self.subTest(head=head, tail=tail, symbol=symbol):
+                    self.assertValid(head + newline + tail, symbol,
+                                     compiler=compiler)
+
+        for tail in ('# comment', ' # comment'):
+            self.assertValid('if 1: pass' + newline + tail, compiler=compiler)
+            self.assertValid('1' + newline + tail, 'eval', compiler=compiler)
+
+        for head in ('1', 'if 1: pass', 'if 1:\n pass'):
+            for tail in (' ', '\t', '\f', '# comment'):
+                self.assertValid(head + newline + tail, 'exec',
+                                 compiler=compiler)
+
     @subTests('symbol', ('single', 'exec', 'eval'))
     def test_unfinished_token_in_invalid_position(self, symbol):
         for source in ('a b', '(a b', '1 """', '(1 """',

Parser/lexer/layout.c

@@ -22,7 +22,7 @@ _PyLexer_ContinueLine(struct tok_state *tok)
     }
     c = tok_nextc(tok);
     if (tok->implicit_newline == _PyTokenizer_APPEND_NEWLINE) {
-        tok->needs_newline = 1;
+        tok->needs_newline |= _PyTokenizer_NEWLINE_CONTINUATION;
     }
     if (c == EOF) {
         tok->done = E_EOF;
@@ -117,6 +117,11 @@ _PyLexer_BeginLine(struct tok_state *tok)
         }
     }
     tok_backup(tok, c);
+    if (c == '\n' && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
+        tok->level == 0 && (cont_line_col || col)) {
+        // Without the newline, a whitespace-only line affects indentation.
+        tok->needs_newline |= _PyTokenizer_NEWLINE_INDENT;
+    }
     if (c == '#' || c == '\n' || c == '\r') {
         int interactive = _PyTok_ReaderIsInteractive(tok);
         /* Lines with only whitespace and/or comments
@@ -174,6 +179,10 @@ _PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
 int
 _PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
 {
+    if (!blankline && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE) {
+        // The statement's terminating newline was not in the input.
+        tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
+    }
     tok->layout.at_bol = 1;
     if (blankline || tok->level > 0) {
         if (!tok->tok_extra_tokens) {

Parser/lexer/lexer.c

@@ -235,6 +235,11 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
                 return type;
             }
         }
+        if (tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
+            tok->layout.depth > 0) {
+            // Ending this comment lets the next line dedent the suite.
+            tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
+        }
         if (tok->tok_extra_tokens) {
             tok_backup(tok, c);  /* don't eat the newline or EOF */
             p_start = comment;

Parser/pegen.c

@@ -1247,10 +1247,12 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
 
     result = _PyPegen_run_parser(p);
     if (result != NULL && (parser_flags & PyPARSE_ALLOW_INCOMPLETE_INPUT)) {
-        _PyTokenizer_Info info = _PyTokenizer_GetInfo(tok);
-        if (info.needs_newline ||
+        int needs_newline = _PyTokenizer_GetNewlineRequirements(tok);
+        if ((needs_newline & _PyTokenizer_NEWLINE_CONTINUATION) ||
+            (start_rule != Py_file_input &&
+             (needs_newline & _PyTokenizer_NEWLINE_INDENT)) ||
             (start_rule == Py_single_input && p->input_exhausted &&
-             info.implicit_newline == _PyTokenizer_IMPLY_NEWLINE)) {
+             (needs_newline & _PyTokenizer_NEWLINE_STATEMENT))) {
             _PyPegen_raise_incomplete_input(p);
             if (incomplete != NULL && PyErr_ExceptionMatches(PyExc_IncompleteInputError)) {
                 *incomplete = PyErr_GetRaisedException();

Parser/pegen.h

@@ -88,8 +88,8 @@ typedef struct {
     int n_keyword_lists;
     int start_rule;
     int *errcode;
-    int parsing_started;
-    int input_exhausted;
+    unsigned char parsing_started;
+    unsigned char input_exhausted;
     PyObject* normalize;
     int starting_lineno;
     int starting_col_offset;

Parser/tokenizer/api.c

@@ -13,13 +13,18 @@ _PyTokenizer_GetStatus(const struct tok_state *tok)
     return tok->done;
 }
 
+int
+_PyTokenizer_GetNewlineRequirements(const struct tok_state *tok)
+{
+    return tok->needs_newline;
+}
+
 _PyTokenizer_Info
 _PyTokenizer_GetInfo(const struct tok_state *tok)
 {
     _PyTokenizer_Info info = {
         .status = tok->done,
         .implicit_newline = tok->implicit_newline,
-        .needs_newline = tok->needs_newline,
         .diagnostic = tok->diagnostic,
         .location = {tok->lineno, tok->line_start < 0
             ? -1 : (int)(tok->cur - tok->line_start)},

Parser/tokenizer/tokenizer.h

@@ -12,6 +12,13 @@ typedef enum {
     _PyTokenizer_APPEND_NEWLINE, // Also append a blank line to terminated input.
 } _PyTokenizer_Newline;
 
+/* Reasons that the appended newline can change the token stream. */
+enum {
+    _PyTokenizer_NEWLINE_STATEMENT = 1,
+    _PyTokenizer_NEWLINE_INDENT = 2,
+    _PyTokenizer_NEWLINE_CONTINUATION = 4,
+};
+
 /* Initialize before use. metadata owns a reference released by _PyToken_Free;
    a consumer taking that reference must set metadata to NULL. */
 struct token {
@@ -44,7 +51,6 @@ typedef struct {
 typedef struct {
     int status;
     unsigned char implicit_newline;  // _PyTokenizer_Newline
-    unsigned char needs_newline;
     _PyTokenizer_Diagnostic diagnostic;
     _PyTok_Loc location;
     _PyTok_Off cursor;
@@ -80,6 +86,7 @@ _PyToken_Free(struct token *token)
    mutated or freed. Source spans may be discarded when reading more input. */
 _PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);
 int _PyTokenizer_GetStatus(const struct tok_state *);
+int _PyTokenizer_GetNewlineRequirements(const struct tok_state *);
 /* An absent token span has a nonnull empty text view. */
 const char *_PyToken_TextView(
     const struct tok_state *, const struct token *, Py_ssize_t *);