[ruby/irb] Use error tokens if there are no correct tokens in the same place

For example, the broken code "%www" will result in only one error token.

https://github.com/ruby/irb/commit/9fa39a7cf3

lib/irb/ruby-lex.rb

@@ -66,7 +66,6 @@ class RubyLex
         unprocessed_tokens = []
         line_num_offset = 0
         tokens.each do |t|
-          next if t[1] == :on_parse_error || t[1] == :compile_error
           partial_tokens << t
           unprocessed_tokens << t
           if t[2].include?("\n")
@@ -107,13 +106,35 @@ class RubyLex
     end
   end
 
+  ERROR_TOKENS = [
+    :on_parse_error,
+    :compile_error,
+    :on_assign_error,
+    :on_alias_error,
+    :on_class_name_error,
+    :on_param_error
+  ]
+
   def ripper_lex_without_warning(code)
     verbose, $VERBOSE = $VERBOSE, nil
     tokens = nil
     self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
       lexer = Ripper::Lexer.new(inner_code, '-', line_no)
       if lexer.respond_to?(:scan) # Ruby 2.7+
-        tokens = lexer.scan
+        tokens = []
+        pos_to_index = {}
+        lexer.scan.each do |t|
+          if pos_to_index.has_key?(t[0])
+            index = pos_to_index[t[0]]
+            found_tk = tokens[index]
+            if ERROR_TOKENS.include?(found_tk[1]) && !ERROR_TOKENS.include?(t[1])
+              tokens[index] = t
+            end
+          else
+            pos_to_index[t[0]] = tokens.size
+            tokens << t
+          end
+        end
       else
         tokens = lexer.parse
       end
@@ -128,7 +149,6 @@ class RubyLex
     prev_spaces = md.nil? ? 0 : md[1].count(' ')
     line_count = 0
     @tokens.each_with_index do |t, i|
-      next if t[1] == :on_parse_error || t[1] == :compile_error
       if t[2].include?("\n")
         line_count += t[2].count("\n")
         if line_count >= line_index
@@ -357,7 +377,6 @@ class RubyLex
     indent = 0
     in_oneliner_def = nil
     tokens.each_with_index { |t, index|
-      next if t[1] == :on_parse_error || t[1] == :compile_error
       # detecting one-liner method definition
       if in_oneliner_def.nil?
         if t[3].allbits?(Ripper::EXPR_ENDFN)
@@ -443,7 +462,6 @@ class RubyLex
     open_brace_on_line = 0
     in_oneliner_def = nil
     @tokens.each_with_index do |t, index|
-      next if t[1] == :on_parse_error || t[1] == :compile_error
       # detecting one-liner method definition
       if in_oneliner_def.nil?
         if t[3].allbits?(Ripper::EXPR_ENDFN)
@@ -513,7 +531,6 @@ class RubyLex
     open_brace_on_line = 0
     in_oneliner_def = nil
     @tokens.each_with_index do |t, index|
-      next if t[1] == :on_parse_error || t[1] == :compile_error
       # detecting one-liner method definition
       if in_oneliner_def.nil?
         if t[3].allbits?(Ripper::EXPR_ENDFN)

test/irb/test_ruby_lex.rb

@@ -442,5 +442,37 @@ module TestIRB
       expected_prompt_list = input_with_prompt.map(&:prompt)
       assert_dynamic_prompt(lines, expected_prompt_list)
     end
+
+    def test_broken_percent_literal
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        skip 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
+
+      ruby_lex = RubyLex.new
+      tokens = ruby_lex.ripper_lex_without_warning('%wwww')
+      pos_to_index = {}
+      tokens.each_with_index { |t, i|
+        assert_nil(pos_to_index[t[0]], "There is already another token in the position of #{t.inspect}.")
+        pos_to_index[t[0]] = i
+      }
+    end
+
+    def test_broken_percent_literal_in_method
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        skip 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
+
+      ruby_lex = RubyLex.new
+      tokens = ruby_lex.ripper_lex_without_warning(<<~EOC.chomp)
+        def foo
+          %wwww
+        end
+      EOC
+      pos_to_index = {}
+      tokens.each_with_index { |t, i|
+        assert_nil(pos_to_index[t[0]], "There is already another token in the position of #{t.inspect}.")
+        pos_to_index[t[0]] = i
+      }
+    end
   end
 end