[ruby/irb] Simplify each_top_level_statement

(https://github.com/ruby/irb/pull/576)

* Simplify each_top_level_statement, reduce instance vars

* Update lib/irb/ruby-lex.rb

Co-authored-by: Stan Lo <stan001212@gmail.com>

* Remove unused ltype from TestRubyLex#check_state response

* Remove unnecessary const path of TerminateLineInput

* Combine duplicated code state check into method

---------

https://github.com/ruby/irb/commit/172453cec4

Co-authored-by: Stan Lo <stan001212@gmail.com>

lib/irb/ruby-lex.rb

@@ -18,10 +18,7 @@ class RubyLex
 
   def initialize(context)
     @context = context
-    @exp_line_no = @line_no = 1
-    @indent = 0
-    @continue = false
-    @line = ""
+    @line_no = 1
     @prompt = nil
   end
 
@@ -42,6 +39,11 @@ class RubyLex
     result
   end
 
+  def single_line_command?(code)
+    command = code.split(/\s/, 2).first
+    @context.symbol_alias?(command) || @context.transform_args?(command)
+  end
+
   # io functions
   def set_input(&block)
     @input = block
@@ -65,14 +67,9 @@ class RubyLex
           end
         else
           # Accept any single-line input for symbol aliases or commands that transform args
-          command = code.split(/\s/, 2).first
-          if @context.symbol_alias?(command) || @context.transform_args?(command)
-            next true
-          end
+          next true if single_line_command?(code)
 
-          code.gsub!(/\s*\z/, '').concat("\n")
-          tokens = self.class.ripper_lex_without_warning(code, context: @context)
-          ltype, indent, continue, code_block_open = check_state(code, tokens)
+          ltype, indent, continue, code_block_open = check_code_state(code)
           if ltype or indent > 0 or continue or code_block_open
             false
           else
@@ -210,69 +207,58 @@ class RubyLex
     [ltype, indent, continue, code_block_open]
   end
 
-  def prompt
-    if @prompt
-      @prompt.call(@ltype, @indent, @continue, @line_no)
-    end
+  def check_code_state(code)
+    check_target_code = code.gsub(/\s*\z/, '').concat("\n")
+    tokens = self.class.ripper_lex_without_warning(check_target_code, context: @context)
+    check_state(check_target_code, tokens)
   end
 
-  def initialize_input
-    @ltype = nil
-    @indent = 0
-    @continue = false
-    @line = ""
-    @exp_line_no = @line_no
-    @code_block_open = false
+  def save_prompt_to_context_io(ltype, indent, continue, line_num_offset)
+    # Implicitly saves prompt string to `@context.io.prompt`. This will be used in the next `@input.call`.
+    @prompt.call(ltype, indent, continue, @line_no + line_num_offset)
+  end
+
+  def readmultiline
+    save_prompt_to_context_io(nil, 0, false, 0)
+
+    # multiline
+    return @input.call if @io.respond_to?(:check_termination)
+
+    # nomultiline
+    code = ''
+    line_offset = 0
+    loop do
+      line = @input.call
+      unless line
+        return code.empty? ? nil : code
+      end
+
+      code << line
+      # Accept any single-line input for symbol aliases or commands that transform args
+      return code if single_line_command?(code)
+
+      ltype, indent, continue, code_block_open = check_code_state(code)
+      return code unless ltype or indent > 0 or continue or code_block_open
+
+      line_offset += 1
+      save_prompt_to_context_io(ltype, indent, continue, line_offset)
+    end
   end
 
   def each_top_level_statement
-    initialize_input
-    catch(:TERM_INPUT) do
-      loop do
-        begin
-          prompt
-          unless l = lex
-            throw :TERM_INPUT if @line == ''
-          else
-            @line_no += l.count("\n")
-            if l == "\n"
-              @exp_line_no += 1
-              next
-            end
-            @line.concat l
-            if @code_block_open or @ltype or @continue or @indent > 0
-              next
-            end
-          end
-          if @line != "\n"
-            @line.force_encoding(@io.encoding)
-            yield @line, @exp_line_no
-          end
-          raise TerminateLineInput if @io.eof?
-          @line = ''
-          @exp_line_no = @line_no
+    loop do
+      code = readmultiline
+      break unless code
 
-          @indent = 0
-        rescue TerminateLineInput
-          initialize_input
-          prompt
-        end
+      if code != "\n"
+        code.force_encoding(@io.encoding)
+        yield code, @line_no
       end
+      @line_no += code.count("\n")
+    rescue TerminateLineInput
     end
   end
 
-  def lex
-    line = @input.call
-    if @io.respond_to?(:check_termination)
-      return line # multiline
-    end
-    code = @line + (line.nil? ? '' : line)
-    code.gsub!(/\s*\z/, '').concat("\n")
-    @tokens = self.class.ripper_lex_without_warning(code, context: @context)
-    @ltype, @indent, @continue, @code_block_open = check_state(code, @tokens)
-    line
-  end
-
   def process_continue(tokens)
     # last token is always newline
     if tokens.size >= 2 and tokens[-2].event == :on_regexp_end

test/irb/test_ruby_lex.rb

@@ -83,27 +83,22 @@ module TestIRB
     end
 
     def assert_nesting_level(lines, expected, local_variables: [])
-      ruby_lex = ruby_lex_for_lines(lines, local_variables: local_variables)
+      indent, _code_block_open = check_state(lines, local_variables: local_variables)
       error_message = "Calculated the wrong number of nesting level for:\n #{lines.join("\n")}"
-      assert_equal(expected, ruby_lex.instance_variable_get(:@indent), error_message)
+      assert_equal(expected, indent, error_message)
     end
 
     def assert_code_block_open(lines, expected, local_variables: [])
-      ruby_lex = ruby_lex_for_lines(lines, local_variables: local_variables)
+      _indent, code_block_open = check_state(lines, local_variables: local_variables)
       error_message = "Wrong result of code_block_open for:\n #{lines.join("\n")}"
-      assert_equal(expected, ruby_lex.instance_variable_get(:@code_block_open), error_message)
+      assert_equal(expected, code_block_open, error_message)
     end
 
-    def ruby_lex_for_lines(lines, local_variables: [])
+    def check_state(lines, local_variables: [])
       context = build_context(local_variables)
       ruby_lex = RubyLex.new(context)
-
-      io = proc{ lines.join("\n") }
-      ruby_lex.set_input do
-        lines.join("\n")
-      end
-      ruby_lex.lex
-      ruby_lex
+      _ltype, indent, _continue, code_block_open = ruby_lex.check_code_state(lines.join("\n"))
+      [indent, code_block_open]
     end
 
     def test_auto_indent