[ruby/irb] Store context in RubyLex

Some background for this refactor:

1. Through a RubyLex instance's lifetime, the context passed to its methods
   should be the same.
   Given that `Context` is only initialised in `Irb#initialize`,
   this should be true.

2. When `RubyLex` is initialised, the context object should be accessible.
   This is also true in all 3 of `RubyLex.new`'s invocations.

With the above observations, we should be able to store the context in `RubyLex`
as an instance variable. And doing so will make `RubyLex`'s instance methods
easier to use and maintain.

https://github.com/ruby/irb/commit/5c8d3df2df

lib/irb.rb

@@ -468,7 +468,7 @@ module IRB
       @context = Context.new(self, workspace, input_method)
       @context.main.extend ExtendCommandBundle
       @signal_status = :IN_IRB
-      @scanner = RubyLex.new
+      @scanner = RubyLex.new(@context)
     end
 
     # A hook point for `debug` command's TracePoint after :IRB_EXIT as well as its clean-up
@@ -538,7 +538,7 @@ module IRB
         @context.io.prompt
       end
 
-      @scanner.set_input(@context.io, context: @context) do
+      @scanner.set_input(@context.io) do
         signal_status(:IN_INPUT) do
           if l = @context.io.gets
             print l if @context.verbose?
@@ -556,9 +556,9 @@ module IRB
         end
       end
 
-      @scanner.set_auto_indent(@context) if @context.auto_indent_mode
+      @scanner.set_auto_indent
 
-      @scanner.each_top_level_statement(@context) do |line, line_no|
+      @scanner.each_top_level_statement do |line, line_no|
         signal_status(:IN_EVAL) do
           begin
             line.untaint if RUBY_VERSION < '2.7'

lib/irb/cmd/show_source.rb

@@ -40,15 +40,15 @@ module IRB
             file, line = receiver.method(method).source_location if receiver.respond_to?(method)
           end
           if file && line
-            Source.new(file: file, first_line: line, last_line: find_end(file, line))
+            Source.new(file: file, first_line: line, last_line: find_end(file, line, irb_context))
           end
         end
 
         private
 
-        def find_end(file, first_line)
+        def find_end(file, first_line, irb_context)
           return first_line unless File.exist?(file)
-          lex = RubyLex.new
+          lex = RubyLex.new(irb_context)
           lines = File.read(file).lines[(first_line - 1)..-1]
           tokens = RubyLex.ripper_lex_without_warning(lines.join)
           prev_tokens = []

lib/irb/ruby-lex.rb

@@ -16,7 +16,8 @@ class RubyLex
     end
   end
 
-  def initialize
+  def initialize(context)
+    @context = context
     @exp_line_no = @line_no = 1
     @indent = 0
     @continue = false
@@ -42,13 +43,13 @@ class RubyLex
   end
 
   # io functions
-  def set_input(io, context:, &block)
+  def set_input(io, &block)
     @io = io
     if @io.respond_to?(:check_termination)
       @io.check_termination do |code|
         if Reline::IOGate.in_pasting?
-          lex = RubyLex.new
-          rest = lex.check_termination_in_prev_line(code, context: context)
+          lex = RubyLex.new(@context)
+          rest = lex.check_termination_in_prev_line(code)
           if rest
             Reline.delete_text
             rest.bytes.reverse_each do |c|
@@ -61,13 +62,13 @@ class RubyLex
         else
           # Accept any single-line input for symbol aliases or commands that transform args
           command = code.split(/\s/, 2).first
-          if context.symbol_alias?(command) || context.transform_args?(command)
+          if @context.symbol_alias?(command) || @context.transform_args?(command)
             next true
           end
 
           code.gsub!(/\s*\z/, '').concat("\n")
-          tokens = self.class.ripper_lex_without_warning(code, context: context)
-          ltype, indent, continue, code_block_open = check_state(code, tokens, context: context)
+          tokens = self.class.ripper_lex_without_warning(code, context: @context)
+          ltype, indent, continue, code_block_open = check_state(code, tokens)
           if ltype or indent > 0 or continue or code_block_open
             false
           else
@@ -80,7 +81,7 @@ class RubyLex
       @io.dynamic_prompt do |lines|
         lines << '' if lines.empty?
         result = []
-        tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: context)
+        tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context)
         code = String.new
         partial_tokens = []
         unprocessed_tokens = []
@@ -93,7 +94,7 @@ class RubyLex
             t_str.each_line("\n") do |s|
               code << s
               next unless s.include?("\n")
-              ltype, indent, continue, code_block_open = check_state(code, partial_tokens, context: context)
+              ltype, indent, continue, code_block_open = check_state(code, partial_tokens)
               result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
               line_num_offset += 1
             end
@@ -104,7 +105,7 @@ class RubyLex
         end
 
         unless unprocessed_tokens.empty?
-          ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens, context: context)
+          ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens)
           result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
         end
         result
@@ -187,11 +188,11 @@ class RubyLex
     prev_spaces
   end
 
-  def set_auto_indent(context)
-    if @io.respond_to?(:auto_indent) and context.auto_indent_mode
+  def set_auto_indent
+    if @io.respond_to?(:auto_indent) and @context.auto_indent_mode
       @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
         if is_newline
-          @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: context)
+          @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: @context)
           prev_spaces = find_prev_spaces(line_index)
           depth_difference = check_newline_depth_difference
           depth_difference = 0 if depth_difference < 0
@@ -200,18 +201,18 @@ class RubyLex
           code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
           last_line = lines[line_index]&.byteslice(0, byte_pointer)
           code += last_line if last_line
-          @tokens = self.class.ripper_lex_without_warning(code, context: context)
+          @tokens = self.class.ripper_lex_without_warning(code, context: @context)
           check_corresponding_token_depth(lines, line_index)
         end
       end
     end
   end
 
-  def check_state(code, tokens, context:)
+  def check_state(code, tokens)
     ltype = process_literal_type(tokens)
     indent = process_nesting_level(tokens)
     continue = process_continue(tokens)
-    lvars_code = self.class.generate_local_variables_assign_code(context.local_variables)
+    lvars_code = self.class.generate_local_variables_assign_code(@context.local_variables)
     code = "#{lvars_code}\n#{code}" if lvars_code
     code_block_open = check_code_block(code, tokens)
     [ltype, indent, continue, code_block_open]
@@ -232,13 +233,13 @@ class RubyLex
     @code_block_open = false
   end
 
-  def each_top_level_statement(context)
+  def each_top_level_statement
     initialize_input
     catch(:TERM_INPUT) do
       loop do
         begin
           prompt
-          unless l = lex(context)
+          unless l = lex
             throw :TERM_INPUT if @line == ''
           else
             @line_no += l.count("\n")
@@ -268,15 +269,15 @@ class RubyLex
     end
   end
 
-  def lex(context)
+  def lex
     line = @input.call
     if @io.respond_to?(:check_termination)
       return line # multiline
     end
     code = @line + (line.nil? ? '' : line)
     code.gsub!(/\s*\z/, '').concat("\n")
-    @tokens = self.class.ripper_lex_without_warning(code, context: context)
-    @ltype, @indent, @continue, @code_block_open = check_state(code, @tokens, context: context)
+    @tokens = self.class.ripper_lex_without_warning(code, context: @context)
+    @ltype, @indent, @continue, @code_block_open = check_state(code, @tokens)
     line
   end
 
@@ -777,8 +778,8 @@ class RubyLex
     end
   end
 
-  def check_termination_in_prev_line(code, context:)
-    tokens = self.class.ripper_lex_without_warning(code, context: context)
+  def check_termination_in_prev_line(code)
+    tokens = self.class.ripper_lex_without_warning(code, context: @context)
     past_first_newline = false
     index = tokens.rindex do |t|
       # traverse first token before last line

test/irb/test_ruby_lex.rb

@@ -36,13 +36,13 @@ module TestIRB
 
       context = build_context
       context.auto_indent_mode = true
-      ruby_lex = RubyLex.new()
+      ruby_lex = RubyLex.new(context)
       io = MockIO_AutoIndent.new([lines, last_line_index, byte_pointer, add_new_line]) do |auto_indent|
         error_message = "Calculated the wrong number of spaces for:\n #{lines.join("\n")}"
         assert_equal(correct_space_count, auto_indent, error_message)
       end
-      ruby_lex.set_input(io, context: context)
-      ruby_lex.set_auto_indent(context)
+      ruby_lex.set_input(io)
+      ruby_lex.set_auto_indent
     end
 
     def assert_nesting_level(lines, expected, local_variables: [])
@@ -58,14 +58,14 @@ module TestIRB
     end
 
     def ruby_lex_for_lines(lines, local_variables: [])
-      ruby_lex = RubyLex.new()
-
       context = build_context(local_variables)
+      ruby_lex = RubyLex.new(context)
+
       io = proc{ lines.join("\n") }
-      ruby_lex.set_input(io, context: context) do
+      ruby_lex.set_input(io) do
         lines.join("\n")
       end
-      ruby_lex.lex(context)
+      ruby_lex.lex
       ruby_lex
     end
 
@@ -633,7 +633,8 @@ module TestIRB
 
     def assert_dynamic_prompt(lines, expected_prompt_list)
       pend if RUBY_ENGINE == 'truffleruby'
-      ruby_lex = RubyLex.new()
+      context = build_context
+      ruby_lex = RubyLex.new(context)
       io = MockIO_DynamicPrompt.new(lines) do |prompt_list|
         error_message = <<~EOM
           Expected dynamic prompt:
@@ -647,8 +648,7 @@ module TestIRB
       ruby_lex.set_prompt do |ltype, indent, continue, line_no|
         '%03d:%01d:%1s:%s ' % [line_no, indent, ltype, continue ? '*' : '>']
       end
-      context = build_context
-      ruby_lex.set_input(io, context: context)
+      ruby_lex.set_input(io)
     end
 
     def test_dyanmic_prompt
@@ -751,9 +751,10 @@ module TestIRB
     end
 
     def test_unterminated_heredoc_string_literal
+      context = build_context
       ['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
         tokens = RubyLex.ripper_lex_without_warning(code)
-        string_literal = RubyLex.new.check_string_literal(tokens)
+        string_literal = RubyLex.new(context).check_string_literal(tokens)
         assert_equal('<<A', string_literal&.tok)
       end
     end
@@ -779,8 +780,9 @@ module TestIRB
           p(
           )
       EOC
+      context = build_context
       [reference_code, code_with_heredoc, code_with_embdoc].each do |code|
-        lex = RubyLex.new
+        lex = RubyLex.new(context)
         lines = code.lines
         lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code))
         assert_equal 2, lex.check_corresponding_token_depth(lines, lines.size)
@@ -788,7 +790,7 @@ module TestIRB
     end
 
     def test_find_prev_spaces_with_multiline_literal
-      lex = RubyLex.new
+      lex = RubyLex.new(build_context)
       reference_code = <<~EOC.chomp
         if true
           1
@@ -813,8 +815,9 @@ module TestIRB
           world
         end
       EOC
+      context = build_context
       [reference_code, code_with_percent_string, code_with_quoted_string].each do |code|
-        lex = RubyLex.new
+        lex = RubyLex.new(context)
         lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code))
         prev_spaces = (1..code.lines.size).map { |index| lex.find_prev_spaces index }
         assert_equal [0, 2, 2, 2, 2, 0], prev_spaces