From b43cc51dcad9859ea6c54cb4f03105c8511582de Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Wed, 4 Oct 2023 13:13:27 +0100 Subject: [PATCH] [ruby/irb] Clear all context usages in RubyLex (https://github.com/ruby/irb/pull/684) After this change, `RubyLex` will not interact with `Context` directly in any way. This decoupling has a few benefits: - It makes `RubyLex` easier to test as it no longer has a dependency on `Context`. We can see this from the removal of `build_context` from `test_ruby_lex.rb`. - It will make `RubyLex` easier to understand as it will not be affected by state changes in `Context` objects. - It allows `RubyLex` to be used in places where `Context` is not available. https://github.com/ruby/irb/commit/d5b262a076 --- lib/irb.rb | 15 ++++++++------- lib/irb/ruby-lex.rb | 31 +++++++++++++++---------------- lib/irb/source_finder.rb | 4 ++-- test/irb/test_irb.rb | 2 +- test/irb/test_ruby_lex.rb | 36 +++++++++--------------------------- 5 files changed, 35 insertions(+), 53 deletions(-) diff --git a/lib/irb.rb b/lib/irb.rb index 57ec911f34..ba7feabf06 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -436,7 +436,7 @@ module IRB @context = Context.new(self, workspace, input_method) @context.workspace.load_commands_to_main @signal_status = :IN_IRB - @scanner = RubyLex.new(@context) + @scanner = RubyLex.new end # A hook point for `debug` command's breakpoint after :IRB_EXIT as well as its clean-up @@ -610,7 +610,7 @@ module IRB # Accept any single-line input for symbol aliases or commands that transform args return code if single_line_command?(code) - tokens, opens, terminated = @scanner.check_code_state(code) + tokens, opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables) return code if terminated line_offset += 1 @@ -643,7 +643,8 @@ module IRB if command_class Statement::Command.new(code, command, arg, command_class) else - Statement::Expression.new(code, @scanner.assignment_expression?(code)) + is_assignment_expression = @scanner.assignment_expression?(code, local_variables: @context.local_variables) + Statement::Expression.new(code, is_assignment_expression) end end @@ -656,7 +657,7 @@ module IRB if @context.io.respond_to?(:check_termination) @context.io.check_termination do |code| if Reline::IOGate.in_pasting? - rest = @scanner.check_termination_in_prev_line(code) + rest = @scanner.check_termination_in_prev_line(code, local_variables: @context.local_variables) if rest Reline.delete_text rest.bytes.reverse_each do |c| @@ -670,7 +671,7 @@ module IRB # Accept any single-line input for symbol aliases or commands that transform args next true if single_line_command?(code) - _tokens, _opens, terminated = @scanner.check_code_state(code) + _tokens, _opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables) terminated end end @@ -678,7 +679,7 @@ module IRB if @context.io.respond_to?(:dynamic_prompt) @context.io.dynamic_prompt do |lines| lines << '' if lines.empty? - tokens = RubyLex.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context) + tokens = RubyLex.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, local_variables: @context.local_variables) line_results = IRB::NestingParser.parse_by_line(tokens) tokens_until_line = [] line_results.map.with_index do |(line_tokens, _prev_opens, next_opens, _min_depth), line_num_offset| @@ -698,7 +699,7 @@ module IRB next nil if !is_newline && lines[line_index]&.byteslice(0, byte_pointer)&.match?(/\A\s*\z/) code = lines[0..line_index].map { |l| "#{l}\n" }.join - tokens = RubyLex.ripper_lex_without_warning(code, context: @context) + tokens = RubyLex.ripper_lex_without_warning(code, local_variables: @context.local_variables) @scanner.process_indent_level(tokens, lines, line_index, is_newline) end end diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index 502883bd43..63756d8f80 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -44,8 +44,7 @@ module IRB attr_reader :line_no - def initialize(context) - @context = context + def initialize @line_no = 1 @prompt = nil end @@ -116,9 +115,9 @@ module IRB interpolated end - def self.ripper_lex_without_warning(code, context: nil) + def self.ripper_lex_without_warning(code, local_variables: []) verbose, $VERBOSE = $VERBOSE, nil - lvars_code = generate_local_variables_assign_code(context&.local_variables || []) + lvars_code = generate_local_variables_assign_code(local_variables) original_code = code if lvars_code code = "#{lvars_code}\n#{code}" @@ -152,14 +151,14 @@ module IRB @prompt&.call(ltype, indent_level, opens.any? || continue, @line_no + line_num_offset) end - def check_code_state(code) - tokens = self.class.ripper_lex_without_warning(code, context: @context) + def check_code_state(code, local_variables:) + tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables) opens = NestingParser.open_tokens(tokens) - [tokens, opens, code_terminated?(code, tokens, opens)] + [tokens, opens, code_terminated?(code, tokens, opens, local_variables: local_variables)] end - def code_terminated?(code, tokens, opens) - case check_code_syntax(code) + def code_terminated?(code, tokens, opens, local_variables:) + case check_code_syntax(code, local_variables: local_variables) when :unrecoverable_error true when :recoverable_error @@ -180,7 +179,7 @@ module IRB @line_no += addition end - def assignment_expression?(code) + def assignment_expression?(code, local_variables:) # Try to parse the code and check if the last of possibly multiple # expressions is an assignment type. @@ -190,7 +189,7 @@ module IRB # array of parsed expressions. The first element of each expression is the # expression's type. verbose, $VERBOSE = $VERBOSE, nil - code = "#{RubyLex.generate_local_variables_assign_code(@context.local_variables) || 'nil;'}\n#{code}" + code = "#{RubyLex.generate_local_variables_assign_code(local_variables) || 'nil;'}\n#{code}" # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part. node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0) ASSIGNMENT_NODE_TYPES.include?(node_type) @@ -222,8 +221,8 @@ module IRB false end - def check_code_syntax(code) - lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables) + def check_code_syntax(code, local_variables:) + lvars_code = RubyLex.generate_local_variables_assign_code(local_variables) code = "#{lvars_code}\n#{code}" begin # check if parser error are available @@ -455,8 +454,8 @@ module IRB end end - def check_termination_in_prev_line(code) - tokens = self.class.ripper_lex_without_warning(code, context: @context) + def check_termination_in_prev_line(code, local_variables:) + tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables) past_first_newline = false index = tokens.rindex do |t| # traverse first token before last line @@ -486,7 +485,7 @@ module IRB tokens_without_last_line = tokens[0..index] code_without_last_line = tokens_without_last_line.map(&:tok).join opens_without_last_line = NestingParser.open_tokens(tokens_without_last_line) - if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line) + if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line, local_variables: local_variables) return last_line_tokens.map(&:tok).join end end diff --git a/lib/irb/source_finder.rb b/lib/irb/source_finder.rb index d196fcddc2..959919e8ac 100644 --- a/lib/irb/source_finder.rb +++ b/lib/irb/source_finder.rb @@ -43,7 +43,7 @@ module IRB private def find_end(file, first_line) - lex = RubyLex.new(@irb_context) + lex = RubyLex.new lines = File.read(file).lines[(first_line - 1)..-1] tokens = RubyLex.ripper_lex_without_warning(lines.join) prev_tokens = [] @@ -53,7 +53,7 @@ module IRB code = lines[0..lnum].join prev_tokens.concat chunk continue = lex.should_continue?(prev_tokens) - syntax = lex.check_code_syntax(code) + syntax = lex.check_code_syntax(code, local_variables: []) if !continue && syntax == :valid return first_line + lnum end diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index 4e5a94b1be..4870f35f39 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -584,7 +584,7 @@ module TestIRB def assert_indent_level(lines, expected) code = lines.map { |l| "#{l}\n" }.join # code should end with "\n" - _tokens, opens, _ = @irb.scanner.check_code_state(code) + _tokens, opens, _ = @irb.scanner.check_code_state(code, local_variables: []) indent_level = @irb.scanner.calc_indent_level(opens) error_message = "Calculated the wrong number of indent level for:\n #{lines.join("\n")}" assert_equal(expected, indent_level, error_message) diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb index b9e55c7330..5cfd81dbe8 100644 --- a/test/irb/test_ruby_lex.rb +++ b/test/irb/test_ruby_lex.rb @@ -149,8 +149,7 @@ module TestIRB end def test_assignment_expression - context = build_context - ruby_lex = IRB::RubyLex.new(context) + ruby_lex = IRB::RubyLex.new [ "foo = bar", @@ -173,7 +172,7 @@ module TestIRB "foo\nfoo = bar", ].each do |exp| assert( - ruby_lex.assignment_expression?(exp), + ruby_lex.assignment_expression?(exp, local_variables: []), "#{exp.inspect}: should be an assignment expression" ) end @@ -186,20 +185,18 @@ module TestIRB "foo = bar\nfoo", ].each do |exp| refute( - ruby_lex.assignment_expression?(exp), + ruby_lex.assignment_expression?(exp, local_variables: []), "#{exp.inspect}: should not be an assignment expression" ) end end def test_assignment_expression_with_local_variable - context = build_context - ruby_lex = IRB::RubyLex.new(context) + ruby_lex = IRB::RubyLex.new code = "a /1;x=1#/" - refute(ruby_lex.assignment_expression?(code), "#{code}: should not be an assignment expression") - context.workspace.binding.eval('a = 1') - assert(ruby_lex.assignment_expression?(code), "#{code}: should be an assignment expression") - refute(ruby_lex.assignment_expression?(""), "empty code should not be an assignment expression") + refute(ruby_lex.assignment_expression?(code, local_variables: []), "#{code}: should not be an assignment expression") + assert(ruby_lex.assignment_expression?(code, local_variables: [:a]), "#{code}: should be an assignment expression") + refute(ruby_lex.assignment_expression?("", local_variables: [:a]), "empty code should not be an assignment expression") end def test_initialising_the_old_top_level_ruby_lex @@ -211,20 +208,6 @@ module TestIRB private - def build_context(local_variables = nil) - IRB.init_config(nil) - workspace = IRB::WorkSpace.new(TOPLEVEL_BINDING.dup) - - if local_variables - local_variables.each do |n| - workspace.binding.local_variable_set(n, nil) - end - end - - IRB.conf[:VERBOSE] = false - IRB::Context.new(nil, workspace, TestInputMethod.new) - end - def assert_indent_level(lines, expected, local_variables: []) indent_level, _continue, _code_block_open = check_state(lines, local_variables: local_variables) error_message = "Calculated the wrong number of indent level for:\n #{lines.join("\n")}" @@ -244,10 +227,9 @@ module TestIRB end def check_state(lines, local_variables: []) - context = build_context(local_variables) code = lines.map { |l| "#{l}\n" }.join # code should end with "\n" - ruby_lex = IRB::RubyLex.new(context) - tokens, opens, terminated = ruby_lex.check_code_state(code) + ruby_lex = IRB::RubyLex.new + tokens, opens, terminated = ruby_lex.check_code_state(code, local_variables: local_variables) indent_level = ruby_lex.calc_indent_level(opens) continue = ruby_lex.should_continue?(tokens) [indent_level, continue, !terminated]