[ruby/irb] Fix code terminated check with heredoc and backtick (https://github.com/ruby/irb/pull/390)

* Fix backtick method def method call handled as backtick open

* Fix handling heredoc in check_string_literal

* Sort result of lexer.parse by pos in ruby<2.7. It's not sorted when the given code includes heredoc.

* Update lib/irb/ruby-lex.rb

Co-authored-by: Stan Lo <stan001212@gmail.com>

* Update lib/irb/ruby-lex.rb

Co-authored-by: Stan Lo <stan001212@gmail.com>

* Add check_string_literal test for heredoc code that does not end with newline

https://github.com/ruby/irb/commit/44bc712460

Co-authored-by: Stan Lo <stan001212@gmail.com>

lib/irb/ruby-lex.rb

@@ -162,7 +162,7 @@ class RubyLex
           end
         end
       else
-        lexer.parse.reject { |it| it.pos.first == 0 }
+        lexer.parse.reject { |it| it.pos.first == 0 }.sort_by(&:pos)
       end
     end
   ensure
@@ -706,6 +706,7 @@ class RubyLex
     i = 0
     start_token = []
     end_type = []
+    pending_heredocs = []
     while i < tokens.size
       t = tokens[i]
       case t.event
@@ -729,18 +730,27 @@ class RubyLex
           end
         end
       when :on_backtick
-        start_token << t
-        end_type << :on_tstring_end
+        if t.state.allbits?(Ripper::EXPR_BEG)
+          start_token << t
+          end_type << :on_tstring_end
+        end
       when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
         start_token << t
         end_type << :on_tstring_end
       when :on_heredoc_beg
-        start_token << t
-        end_type << :on_heredoc_end
+        pending_heredocs << t
+      end
+
+      if pending_heredocs.any? && t.tok.include?("\n")
+        pending_heredocs.reverse_each do |t|
+          start_token << t
+          end_type << :on_heredoc_end
+        end
+        pending_heredocs = []
       end
       i += 1
     end
-    start_token.last.nil? ? nil : start_token.last
+    pending_heredocs.first || start_token.last
   end
 
   def process_literal_type(tokens = @tokens)

test/irb/test_ruby_lex.rb

@@ -170,6 +170,40 @@ module TestIRB
       assert_dynamic_prompt(lines, expected_prompt_list)
     end
 
+    def test_heredoc_with_embexpr
+      input_with_prompt = [
+        PromptRow.new('001:0:":* ', %q(<<A+%W[#{<<B)),
+        PromptRow.new('002:0:":* ', %q(#{<<C+%W[)),
+        PromptRow.new('003:0:":* ', %q()),
+        PromptRow.new('004:0:":* ', %q(C)),
+        PromptRow.new('005:0:]:* ', %q()),
+        PromptRow.new('006:0:":* ', %q(]})),
+        PromptRow.new('007:0:":* ', %q(})),
+        PromptRow.new('008:0:":* ', %q(A)),
+        PromptRow.new('009:0:]:* ', %q(B)),
+        PromptRow.new('010:0:]:* ', %q(})),
+        PromptRow.new('011:0: :> ', %q(])),
+        PromptRow.new('012:0: :* ', %q()),
+      ]
+
+      lines = input_with_prompt.map(&:content)
+      expected_prompt_list = input_with_prompt.map(&:prompt)
+      assert_dynamic_prompt(lines, expected_prompt_list)
+    end
+
+    def test_backtick_method
+      input_with_prompt = [
+        PromptRow.new('001:0: :> ', %q(self.`(arg))),
+        PromptRow.new('002:0: :* ', %q()),
+        PromptRow.new('003:0: :> ', %q(def `(); end)),
+        PromptRow.new('004:0: :* ', %q()),
+      ]
+
+      lines = input_with_prompt.map(&:content)
+      expected_prompt_list = input_with_prompt.map(&:prompt)
+      assert_dynamic_prompt(lines, expected_prompt_list)
+    end
+
     def test_incomplete_coding_magic_comment
       input_with_correct_indents = [
         Row.new(%q(#coding:u), nil, 0),
@@ -632,5 +666,13 @@ module TestIRB
         assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token')
       end
     end
+
+    def test_unterminated_heredoc_string_literal
+      ['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
+        tokens = RubyLex.ripper_lex_without_warning(code)
+        string_literal = RubyLex.new.check_string_literal(tokens)
+        assert_equal('<<A', string_literal&.tok)
+      end
+    end
   end
 end