diff --git a/prism/prism.c b/prism/prism.c index 4e0d837a7a..6921feac48 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -8831,6 +8831,8 @@ pm_token_buffer_escape(pm_parser_t *parser, pm_token_buffer_t *token_buffer) { const uint8_t *end = parser->current.end - 1; pm_buffer_append_bytes(&token_buffer->buffer, start, (size_t) (end - start)); + + token_buffer->cursor = end; } /** diff --git a/test/prism/parser_test.rb b/test/prism/parser_test.rb index d3bf52d96c..118b7322fe 100644 --- a/test/prism/parser_test.rb +++ b/test/prism/parser_test.rb @@ -45,22 +45,22 @@ module Prism base = File.join(__dir__, "fixtures") # These files are erroring because of the parser gem being wrong. - skip_incorrect = %w[ - embdoc_no_newline_at_end.txt + skip_incorrect = [ + "embdoc_no_newline_at_end.txt" ] # These files are either failing to parse or failing to translate, so we'll # skip them for now. - skip_all = skip_incorrect | %w[ - dash_heredocs.txt - dos_endings.txt - heredocs_with_ignored_newlines.txt - regex.txt - regex_char_width.txt - spanning_heredoc.txt - spanning_heredoc_newlines.txt - tilde_heredocs.txt - unescaping.txt + skip_all = skip_incorrect | [ + "dash_heredocs.txt", + "dos_endings.txt", + "heredocs_with_ignored_newlines.txt", + "regex.txt", + "regex_char_width.txt", + "spanning_heredoc.txt", + "spanning_heredoc_newlines.txt", + "tilde_heredocs.txt", + "unescaping.txt" ] # Not sure why these files are failing on JRuby, but skipping them for now. @@ -70,21 +70,21 @@ module Prism # These files are failing to translate their lexer output into the lexer # output expected by the parser gem, so we'll skip them for now. - skip_tokens = %w[ - comments.txt - constants.txt - endless_range_in_conditional.txt - heredoc_with_comment.txt - heredoc_with_escaped_newline_at_start.txt - heredocs_leading_whitespace.txt - heredocs_nested.txt - heredocs_with_ignored_newlines_and_non_empty.txt - indented_file_end.txt - non_alphanumeric_methods.txt - range_begin_open_inclusive.txt - single_quote_heredocs.txt - strings.txt - xstring.txt + skip_tokens = [ + "comments.txt", + "constants.txt", + "endless_range_in_conditional.txt", + "heredoc_with_comment.txt", + "heredoc_with_escaped_newline_at_start.txt", + "heredocs_leading_whitespace.txt", + "heredocs_nested.txt", + "heredocs_with_ignored_newlines_and_non_empty.txt", + "indented_file_end.txt", + "non_alphanumeric_methods.txt", + "range_begin_open_inclusive.txt", + "single_quote_heredocs.txt", + "strings.txt", + "xstring.txt" ] Dir["*.txt", base: base].each do |name| diff --git a/test/prism/ripper_test.rb b/test/prism/ripper_test.rb index f19e8ddbb6..07238fc3d5 100644 --- a/test/prism/ripper_test.rb +++ b/test/prism/ripper_test.rb @@ -25,30 +25,29 @@ module Prism # Ripper cannot handle named capture groups in regular expressions. "regex.txt", "regex_char_width.txt", - "whitequark/lvar_injecting_match.txt" + "whitequark/lvar_injecting_match.txt", + + # Ripper fails to understand some structures that span across heredocs. + "spanning_heredoc.txt" ] omitted = [ "dos_endings.txt", "heredocs_with_ignored_newlines.txt", + "seattlerb/block_call_dot_op2_brace_block.txt", + "seattlerb/block_command_operation_colon.txt", + "seattlerb/block_command_operation_dot.txt", "seattlerb/heredoc__backslash_dos_format.txt", "seattlerb/heredoc_backslash_nl.txt", "seattlerb/heredoc_nested.txt", "seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt", - "seattlerb/heredoc_squiggly_no_indent.txt", - "spanning_heredoc.txt", "tilde_heredocs.txt", "unparser/corpus/semantic/dstr.txt", "whitequark/dedenting_heredoc.txt", - "whitequark/parser_bug_640.txt", "whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt", "whitequark/parser_slash_slash_n_escaping_in_literals.txt", - "whitequark/slash_newline_in_heredocs.txt", - - "seattlerb/block_call_dot_op2_brace_block.txt", - "seattlerb/block_command_operation_colon.txt", - "seattlerb/block_command_operation_dot.txt", - "whitequark/send_block_chain_cmd.txt" + "whitequark/send_block_chain_cmd.txt", + "whitequark/slash_newline_in_heredocs.txt" ] relatives.each do |relative|