From 8108efd7ab12d19527aa2ee2b43cdeb77001caab Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 5 Aug 2026 21:51:02 -0400 Subject: [PATCH] Revert "Split up newline token" This reverts commit 3dbe592a76cb5e53423f9a3f0ac4840334e783e9. --- config.yml | 2 -- lib/prism/lex_compat.rb | 26 +---------------- lib/prism/translation/parser/lexer.rb | 41 +++++++++++++++++---------- src/prism.c | 30 +++++--------------- templates/src/tokens.c.erb | 2 -- test/prism/ruby/parser_test.rb | 15 +++++++++- 6 files changed, 48 insertions(+), 68 deletions(-) diff --git a/config.yml b/config.yml index 4892089c03..cc5eb7e099 100644 --- a/config.yml +++ b/config.yml @@ -359,8 +359,6 @@ tokens: comment: "when" - name: NEWLINE comment: "a newline character outside of other tokens" - - name: NEWLINE_TERMINATOR - comment: "a newline that terminates a construct where a newline is otherwise insignificant" - name: PARENTHESIS_RIGHT comment: ")" - name: PIPE diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index a2ad69cd29..749f11173a 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -191,7 +191,6 @@ def deconstruct_keys(keys) # :nodoc: MINUS_EQUAL: :on_op, MINUS_GREATER: :on_tlambda, NEWLINE: :on_nl, - NEWLINE_TERMINATOR: :on_ignored_nl, NUMBERED_REFERENCE: :on_backref, PARENTHESIS_LEFT: :on_lparen, PARENTHESIS_LEFT_GROUPING: :on_lparen, @@ -618,9 +617,6 @@ def result bom = source.slice(0, 3) == "\xEF\xBB\xBF" - last_comment_token = nil #: lex_compat_token? - last_comment_end = nil #: Integer? - result_value.each_with_index do |(prism_token, prism_state), index| lineno = prism_token.location.start_line column = prism_token.location.start_column @@ -629,16 +625,6 @@ def result value = prism_token.value lex_state = Translation::Ripper::Lexer::State[prism_state] - # A comment token does not include its terminating newline, but - # ripper's comment value does, so the newline token that directly - # follows a comment is folded back into it. - if last_comment_token && last_comment_end == prism_token.location.start_offset && (event == :on_nl || event == :on_ignored_nl) - last_comment_token[2] += value - last_comment_token = nil - last_comment_end = nil - next - end - # If there's a UTF-8 byte-order mark as the start of the file, then for # certain tokens ripper sets the first token back by 3 bytes. It also # keeps the byte order mark in the first token's value. This is weird, @@ -728,16 +714,11 @@ def result eof_token = prism_token previous_token = result_value[index - 1][0] - # A newline that was folded back into a comment still marks the - # comment boundary for the check below. - comment_boundary = previous_token.type == :COMMENT || - (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2][0].type == :COMMENT && result_value[index - 2][0].location.end_offset == previous_token.location.start_offset) - # If we're at the end of the file and the previous token was a # comment and there is still whitespace after the comment, then # Ripper will append a on_nl token (even though there isn't # necessarily a newline). We mirror that here. - if comment_boundary + if previous_token.type == :COMMENT # If the comment is at the start of a heredoc: <= 0 + next_token, _ = lexed[index] + + is_inline_comment = prev_token&.location&.start_line == token.location.start_line + if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type) + tokens << [:tCOMMENT, [value, location]] + + nl_location = range(token.location.end_offset - 1, token.location.end_offset) + tokens << [:tNL, [nil, nl_location]] + next + elsif is_inline_comment && next_token&.type == :COMMENT + comment_newline_location = range(token.location.end_offset - 1, token.location.end_offset) + elsif comment_newline_location && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type) + tokens << [:tCOMMENT, [value, location]] + tokens << [:tNL, [nil, comment_newline_location]] + comment_newline_location = nil + next + end end when :tNL next_token, _ = lexed[index] @@ -486,10 +501,6 @@ def to_a end end - if comment_newline_location - tokens << [:tNL, [nil, comment_newline_location]] - end - tokens end diff --git a/src/prism.c b/src/prism.c index 0f9168ad6b..566be9e56d 100644 --- a/src/prism.c +++ b/src/prism.c @@ -10212,6 +10212,7 @@ parser_lex(pm_parser_t *parser) { pm_comment_t *comment = parser_comment(parser, PM_COMMENT_INLINE); pm_list_append(&parser->comment_list, (pm_list_node_t *) comment); + if (ending) parser->current.end++; parser->current.type = PM_TOKEN_COMMENT; parser_lex_callback(parser); @@ -10229,16 +10230,7 @@ parser_lex(pm_parser_t *parser) { } } - /* The comment does not include its terminating newline, - * which lexes through the newline handling below as its - * own token. A comment that ends the file has no newline, - * so the newline handling runs without one to emit. */ - if (ending == NULL) { - lexed_comment = true; - } else { - parser->current.start = ending; - parser->current.end = ending + 1; - } + lexed_comment = true; } PRISM_FALLTHROUGH case '\r': @@ -10276,11 +10268,7 @@ parser_lex(pm_parser_t *parser) { break; case PM_IGNORED_NEWLINE_PATTERN: if (parser->pattern_matching_newlines || parser->in_keyword_arg) { - if (!lexed_comment) { - parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR; - parser_lex_callback(parser); - } - + if (!lexed_comment) parser_lex_ignored_newline(parser); lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; parser->current.type = PM_TOKEN_NEWLINE; @@ -10377,15 +10365,11 @@ parser_lex(pm_parser_t *parser) { // If we hit a . after a newline, then we're in a call chain and // we need to return the call operator. if (next_content[0] == '.') { - /* A beginless range on the next line means this - * newline terminates the statement rather than - * continuing a method chain. */ + // To match ripper, we need to emit an ignored newline even though + // it's a real newline in the case that we have a beginless range + // on a subsequent line. if (peek_at(parser, next_content + 1) == '.') { - if (!lexed_comment) { - parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR; - parser_lex_callback(parser); - } - + if (!lexed_comment) parser_lex_ignored_newline(parser); lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; parser->current.type = PM_TOKEN_NEWLINE; diff --git a/templates/src/tokens.c.erb b/templates/src/tokens.c.erb index 6e88d423c2..fb71afe217 100644 --- a/templates/src/tokens.c.erb +++ b/templates/src/tokens.c.erb @@ -275,8 +275,6 @@ pm_token_str(pm_token_type_t token_type) { return "'->'"; case PM_TOKEN_NEWLINE: return "newline"; - case PM_TOKEN_NEWLINE_TERMINATOR: - return "newline"; case PM_TOKEN_NUMBERED_REFERENCE: return "numbered reference"; case PM_TOKEN_PARENTHESIS_LEFT: diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb index 076f84765c..e44bc20d4d 100644 --- a/test/prism/ruby/parser_test.rb +++ b/test/prism/ruby/parser_test.rb @@ -109,13 +109,26 @@ class ParserTest < TestCase # These files are failing to translate their lexer output into the lexer # output expected by the parser gem, so we'll skip them for now. skip_tokens = [ + "dash_heredocs.txt", "embdoc_no_newline_at_end.txt", + "seattlerb/case_in.txt", + "seattlerb/difficult4__leading_dots2.txt", "seattlerb/heredoc_unicode.txt", "seattlerb/parse_line_heredoc.txt", "seattlerb/pct_w_heredoc_interp_nested.txt", + "seattlerb/required_kwarg_no_value.txt", + "seattlerb/TestRubyParserShared.txt", "unparser/corpus/literal/assignment.txt", "unparser/corpus/literal/literal.txt", - "whitequark/forward_arg_with_open_args.txt" + "whitequark/args.txt", + "whitequark/beginless_erange_after_newline.txt", + "whitequark/beginless_irange_after_newline.txt", + "whitequark/forward_arg_with_open_args.txt", + "whitequark/kwarg_no_paren.txt", + "whitequark/multiple_pattern_matches.txt", + "whitequark/newline_in_hash_argument.txt", + "whitequark/pattern_matching_hash.txt", + "whitequark/ruby_bug_9669.txt" ] Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture|