Refine replacement field, comma and string decode error reporting

Add `ParseErrorType::ExpectedIdentifier` for a missing identifier. Treat
only missing expressions and identifiers as making an expression
incomplete, require a complete prefix of the second expression for the
missing comma error, keep string decode errors read before an unknown
token, and report them ahead of an unclosed bracket.

Assisted-by: Claude Code:claude-opus-5-5

Author: youknowone

crates/ruff_python_parser/resources/inline/err/missing_comma_before_invalid_expression.py

@@ -0,0 +1,2 @@
+[a ~]
+[a b +]

crates/ruff_python_parser/src/error.rs

@@ -288,6 +288,8 @@ pub enum ParseErrorType {
     ExpectedImaginaryNumber,
     /// Expected an expression at the current parser location.
     ExpectedExpression,
+    /// Expected an identifier at the current parser location.
+    ExpectedIdentifier,
     /// The parser expected a specific token that was not found.
     ExpectedToken {
         expected: TokenKind,
@@ -579,7 +581,9 @@ impl std::fmt::Display for ParseErrorType {
             ParseErrorType::ExpectedImaginaryNumber => {
                 f.write_str("expected an imaginary number in complex literal pattern")
             }
-            ParseErrorType::ExpectedExpression => f.write_str("invalid syntax"),
+            ParseErrorType::ExpectedExpression | ParseErrorType::ExpectedIdentifier => {
+                f.write_str("invalid syntax")
+            }
             ParseErrorType::UnexpectedIndentation => f.write_str("unexpected indent"),
             ParseErrorType::InvalidAssignmentTarget {
                 kind,

crates/ruff_python_parser/src/parser/expression.rs

@@ -439,10 +439,18 @@ impl<'src> Parser<'src> {
             return;
         }
 
+        // test_err missing_comma_before_invalid_expression
+        // [a ~]
+        // [a b +]
         let checkpoint = self.checkpoint();
+        let errors_before = self.errors.len();
         let second = self.parse_conditional_expression_or_higher();
-        let range = TextRange::new(first.start(), second.end());
+        let second_end = self.complete_prefix_end(&second.expr, errors_before);
         self.rewind(checkpoint);
+        let Some(second_end) = second_end else {
+            return;
+        };
+        let range = TextRange::new(first.start(), second_end);
         self.add_error(
             ParseErrorType::OtherError("invalid syntax. Perhaps you forgot a comma?".to_string()),
             range,
@@ -798,7 +806,7 @@ impl<'src> Parser<'src> {
 
         if self.current_token_kind().is_keyword() {
             // Non-soft keyword
-            self.add_error(ParseErrorType::OtherError("invalid syntax".into()), range);
+            self.add_error(ParseErrorType::ExpectedIdentifier, range);
 
             let text = self.src_text(range);
             let id = self.intern_name(text);
@@ -814,10 +822,7 @@ impl<'src> Parser<'src> {
     }
 
     fn parse_missing_identifier(&mut self) -> ast::Identifier {
-        self.add_error(
-            ParseErrorType::OtherError("invalid syntax".into()),
-            self.current_token_range(),
-        );
+        self.add_error(ParseErrorType::ExpectedIdentifier, self.current_token_range());
 
         ast::Identifier {
             id: Name::empty(),
@@ -2119,7 +2124,12 @@ impl<'src> Parser<'src> {
             return;
         }
         let (error, range) = match self.errors.get(errors_before) {
-            Some(first) if matches!(first.error, ParseErrorType::ExpectedExpression) => {
+            Some(first)
+                if matches!(
+                    first.error,
+                    ParseErrorType::ExpectedExpression | ParseErrorType::ExpectedIdentifier
+                ) =>
+            {
                 match self.complete_prefix_end(expr, errors_before) {
                     None => (
                         InterpolatedStringErrorType::ExpectedExpressionAfterLbrace,
@@ -2148,13 +2158,16 @@ impl<'src> Parser<'src> {
     }
 
     /// Returns the end of the longest prefix of `expr` that is a complete expression, where an
-    /// expression is incomplete if one of the errors after the first `errors_before` errors is
-    /// inside it, or `None` if no prefix is complete.
+    /// expression is incomplete if a missing expression or identifier after the first
+    /// `errors_before` errors is inside it, or `None` if no prefix is complete.
     fn complete_prefix_end(&self, expr: &Expr, errors_before: usize) -> Option<TextSize> {
         let is_complete = |expr: &Expr| {
-            !self.errors[errors_before..]
-                .iter()
-                .any(|error| expr.range().contains_inclusive(error.location.start()))
+            !self.errors[errors_before..].iter().any(|error| {
+                matches!(
+                    error.error,
+                    ParseErrorType::ExpectedExpression | ParseErrorType::ExpectedIdentifier
+                ) && expr.range().contains_inclusive(error.location.start())
+            })
         };
         if is_complete(expr) {
             return Some(expr.end());

crates/ruff_python_parser/src/parser/mod.rs

@@ -885,8 +885,9 @@ impl<'src> Parser<'src> {
             }
         }
 
-        // The lexer already reported an error for the current token, which comes first.
-        if self.at(TokenKind::Unknown) {
+        // The lexer already reported an error for the current token, which comes first. A
+        // string literal that fails to decode was read before it.
+        if self.at(TokenKind::Unknown) && !matches!(error, ParseErrorType::Lexical(_)) {
             return;
         }
 
@@ -2043,8 +2044,17 @@ fn prioritize_tokenizer_error(
         }
         _ => first_start,
     };
+    // An escape sequence error is found when its string is parsed, before the end of the
+    // source shows an unclosed bracket.
+    let escape_error = matches!(
+        first.error,
+        ParseErrorType::Lexical(
+            LexicalErrorType::UnicodeEscapeError { .. } | LexicalErrorType::BytesEscapeError { .. }
+        )
+    );
     let wins = detected <= first_start
         || (!in_interpolated_string
+            && !escape_error
             && match tokenizer_error.error() {
                 LexicalErrorType::UnclosedBracket { .. } => {
                     let bracket_start = tokenizer_error.location().start();

crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_expected_separator_after_expression.py.snap

@@ -557,6 +557,15 @@ Module(
   |
 
 
+  |
+2 | f"{1 +}"
+3 | f"{a, b and}"
+4 | f"{x.}"
+  |     ^ Syntax Error: f-string: expecting '=', or '!', or ':', or '}'
+5 | A tokenizer error up to where the expression fails comes first.
+  |
+
+
   |
 2 | f"{1 +}"
 3 | f"{a, b and}"

crates/ruff_python_parser/tests/snapshots/invalid_syntax@missing_comma_before_invalid_expression.py.snap

@@ -0,0 +1,135 @@
+---
+source: crates/ruff_python_parser/tests/fixtures.rs
+input_file: crates/ruff_python_parser/resources/inline/err/missing_comma_before_invalid_expression.py
+---
+## AST
+
+```
+Module(
+    ModModule {
+        node_index: NodeIndex(None),
+        range: 0..14,
+        body: [
+            Expr(
+                StmtExpr {
+                    node_index: NodeIndex(None),
+                    range: 0..5,
+                    value: List(
+                        ExprList {
+                            node_index: NodeIndex(None),
+                            range: 0..5,
+                            elts: [
+                                Name(
+                                    ExprName {
+                                        node_index: NodeIndex(None),
+                                        range: 1..2,
+                                        id: Name("a"),
+                                        ctx: Load,
+                                    },
+                                ),
+                                UnaryOp(
+                                    ExprUnaryOp {
+                                        node_index: NodeIndex(None),
+                                        range: 3..4,
+                                        op: Invert,
+                                        operand: Name(
+                                            ExprName {
+                                                node_index: NodeIndex(None),
+                                                range: 4..4,
+                                                id: Name(""),
+                                                ctx: Invalid,
+                                            },
+                                        ),
+                                    },
+                                ),
+                            ],
+                            ctx: Load,
+                            runtime_elts: None,
+                        },
+                    ),
+                },
+            ),
+            Expr(
+                StmtExpr {
+                    node_index: NodeIndex(None),
+                    range: 6..13,
+                    value: List(
+                        ExprList {
+                            node_index: NodeIndex(None),
+                            range: 6..13,
+                            elts: [
+                                Name(
+                                    ExprName {
+                                        node_index: NodeIndex(None),
+                                        range: 7..8,
+                                        id: Name("a"),
+                                        ctx: Load,
+                                    },
+                                ),
+                                BinOp(
+                                    ExprBinOp {
+                                        node_index: NodeIndex(None),
+                                        range: 9..12,
+                                        left: Name(
+                                            ExprName {
+                                                node_index: NodeIndex(None),
+                                                range: 9..10,
+                                                id: Name("b"),
+                                                ctx: Load,
+                                            },
+                                        ),
+                                        op: Add,
+                                        right: Name(
+                                            ExprName {
+                                                node_index: NodeIndex(None),
+                                                range: 12..12,
+                                                id: Name(""),
+                                                ctx: Invalid,
+                                            },
+                                        ),
+                                    },
+                                ),
+                            ],
+                            ctx: Load,
+                            runtime_elts: None,
+                        },
+                    ),
+                },
+            ),
+        ],
+        runtime_body: None,
+    },
+)
+```
+## Errors
+
+  |
+1 | [a ~]
+  |    ^ Syntax Error: invalid syntax
+2 | [a b +]
+  |
+
+
+  |
+1 | [a ~]
+  |     ^ Syntax Error: invalid syntax
+2 | [a b +]
+  |
+
+
+  |
+1 | [a ~]
+2 | [a b +]
+  |  ^^^ Syntax Error: invalid syntax. Perhaps you forgot a comma?
+
+
+  |
+1 | [a ~]
+2 | [a b +]
+  |    ^ Syntax Error: invalid syntax
+
+
+  |
+1 | [a ~]
+2 | [a b +]
+  |       ^ Syntax Error: invalid syntax