Introduce another semantic value stack for Ripper so that
Ripper can manage both Node and Ruby Object separately.
This rearchitectutre of Ripper solves these issues.
Therefore adding test cases for them.

* [Bug 10436] https://bugs.ruby-lang.org/issues/10436
* [Bug 18988] https://bugs.ruby-lang.org/issues/18988
* [Bug 20055] https://bugs.ruby-lang.org/issues/20055

Checked the differences of `Ripper.sexp` for files under `/test/ruby`
are only on test_pattern_matching.rb.
The differences comes from the differences between
`new_hash_pattern_tail` functions between parser and Ripper.
Ripper `new_hash_pattern_tail` didn’t call `assignable` then
`kw_rest_arg` wasn’t marked as local variable.
This is also fixed by this commit.

```
--- a/./tmp/before/test_pattern_matching.rb
+++ b/./tmp/after/test_pattern_matching.rb
@@ -3607,7 +3607,7 @@
                  [:in,
                   [:hshptn, nil, [], [:var_field, [:@ident, “a”, [984, 13]]]],
                   [[:binary,
-                    [:vcall, [:@ident, “a”, [985, 10]]],
+                    [:var_ref, [:@ident, “a”, [985, 10]]],
                     :==,
                     [:hash, nil]]],
                   nil]]],
@@ -3662,7 +3662,7 @@
                  [:in,
                   [:hshptn, nil, [], [:var_field, [:@ident, “a”, [993, 13]]]],
                   [[:binary,
-                    [:vcall, [:@ident, “a”, [994, 10]]],
+                    [:var_ref, [:@ident, “a”, [994, 10]]],
                     :==,
                     [:hash,
                      [:assoclist_from_args,
@@ -3813,7 +3813,7 @@
                    [:command,
                     [:@ident, “raise”, [1022, 10]],
                     [:args_add_block,
-                     [[:vcall, [:@ident, “b”, [1022, 16]]]],
+                     [[:var_ref, [:@ident, “b”, [1022, 16]]]],
                      false]]],
                   [:else, [[:var_ref, [:@kw, “true”, [1024, 10]]]]]]]],
                nil,
@@ -3876,7 +3876,7 @@
                      [:@int, “0”, [1033, 15]]],
                     :“&&“,
                     [:binary,
-                     [:vcall, [:@ident, “b”, [1033, 20]]],
+                     [:var_ref, [:@ident, “b”, [1033, 20]]],
                      :==,
                      [:hash, nil]]]],
                   nil]]],
@@ -3946,7 +3946,7 @@
                      [:@int, “0”, [1042, 15]]],
                     :“&&“,
                     [:binary,
-                     [:vcall, [:@ident, “b”, [1042, 20]]],
+                     [:var_ref, [:@ident, “b”, [1042, 20]]],
                      :==,
                      [:hash,
                       [:assoclist_from_args,
@@ -5206,7 +5206,7 @@
                      [[:assoc_new,
                        [:@label, “c:“, [1352, 22]],
                        [:@int, “0”, [1352, 25]]]]]],
-                   [:vcall, [:@ident, “r”, [1352, 29]]]],
+                   [:var_ref, [:@ident, “r”, [1352, 29]]]],
                   false]]],
                [:binary,
                 [:call,
@@ -5299,7 +5299,7 @@
                       [:assoc_new,
                        [:@label, “c:“, [1367, 34]],
                        [:@int, “0”, [1367, 37]]]]]],
-                   [:vcall, [:@ident, “r”, [1367, 41]]]],
+                   [:var_ref, [:@ident, “r”, [1367, 41]]]],
                   false]]],
                [:binary,
                 [:call,
@@ -5931,7 +5931,7 @@
              [:in,
               [:hshptn, nil, [], [:var_field, [:@ident, “r”, [1533, 11]]]],
               [[:binary,
-                [:vcall, [:@ident, “r”, [1534, 8]]],
+                [:var_ref, [:@ident, “r”, [1534, 8]]],
                 :==,
                 [:hash,
                  [:assoclist_from_args,
```
This commit is contained in:
yui-knk 2024-01-12 10:46:17 +09:00 коммит произвёл Yuichiro Kaneko
Родитель f75b9dbf7d
Коммит 89cfc15207
17 изменённых файлов: 1003 добавлений и 1623 удалений

2
ast.c
Просмотреть файл

@ -712,8 +712,6 @@ node_children(rb_ast_t *ast, const NODE *node)
case NODE_ERROR:
return rb_ary_new_from_node_args(ast, 0);
case NODE_ARGS_AUX:
case NODE_RIPPER:
case NODE_RIPPER_VALUES:
case NODE_LAST:
break;
}

Просмотреть файл

@ -54,27 +54,6 @@ static const rb_data_type_t parser_data_type = {
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
};
ID
ripper_get_id(VALUE v)
{
NODE *nd;
if (!RB_TYPE_P(v, T_NODE)) return 0;
nd = (NODE *)v;
if (!nd_type_p(nd, NODE_RIPPER)) return 0;
return RNODE_RIPPER(nd)->nd_vid;
}
VALUE
ripper_get_value(VALUE v)
{
NODE *nd;
if (UNDEF_P(v)) return Qnil;
if (!RB_TYPE_P(v, T_NODE)) return v;
nd = (NODE *)v;
if (!nd_type_p(nd, NODE_RIPPER)) return Qnil;
return RNODE_RIPPER(nd)->nd_rval;
}
static VALUE
ripper_lex_get_generic(struct parser_params *p, VALUE src)
{
@ -607,5 +586,8 @@ InitVM_ripper(void)
*/
rb_define_global_const("SCRIPT_LINES__", Qnil);
#endif
rb_ripper_none = rb_obj_alloc(rb_cObject);
rb_obj_freeze(rb_ripper_none);
rb_gc_register_mark_object(rb_ripper_none);
}

Просмотреть файл

@ -1,6 +1,7 @@
#ifndef RIPPER_INIT_H
#define RIPPER_INIT_H
extern VALUE rb_ripper_none;
VALUE ripper_get_value(VALUE v);
ID ripper_get_id(VALUE v);
PRINTF_ARGS(void ripper_compile_error(struct parser_params*, const char *fmt, ...), 2, 3);

Просмотреть файл

@ -18,14 +18,15 @@ class DSL
NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source
NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source
def initialize(code, options)
def initialize(code, options, lineno = nil)
@lineno = lineno
@events = {}
@error = options.include?("error")
@brace = options.include?("brace")
if options.include?("final")
@final = "p->result"
else
@final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "$$")
@final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue")
end
@vars = 0
@ -33,8 +34,11 @@ class DSL
p = p = "p"
@code = ""
code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K[$@]#{TAG_PATTERN}?#{NAME_PATTERN}]o, '"\&"')
code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o, '"\&"')
@last_value = eval(code)
rescue SyntaxError
$stderr.puts "error on line #{@lineno}" if @lineno
raise
end
attr_reader :events
@ -65,11 +69,15 @@ class DSL
vars = []
args.each do |arg|
vars << v = new_var
@code << "#{ v }=#{ arg };"
if arg =~ /\A\$:#{NAME_PATTERN}\z/
@code << "#{ v }=get_value(#{arg});"
else
@code << "#{ v }=#{ arg };"
end
end
v = new_var
d = "dispatch#{ args.size }(#{ [event, *vars].join(",") })"
d = "#{ vars.last }==Qundef ? #{ vars.first } : #{ d }" if qundef_check
d = "#{ vars.last }==rb_ripper_none ? #{ vars.first } : #{ d }" if qundef_check
@code << "#{ v }=#{ d };"
v
end
@ -88,4 +96,3 @@ class DSL
name
end
end

Просмотреть файл

@ -167,14 +167,14 @@ require_relative "dsl"
def read_ids1_with_locations(path)
h = {}
File.open(path) {|f|
f.each do |line|
f.each.with_index(1) do |line, i|
next if /\A\#\s*define\s+dispatch/ =~ line
next if /ripper_dispatch/ =~ line
line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event|
(h[event] ||= []).push [f.lineno, arity.to_i]
end
if line =~ %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/>
gen = DSL.new($2, ($1 || "").split(","))
gen = DSL.new($2, ($1 || "").split(","), i)
gen.generate
gen.events.each do |event, arity|
(h[event] ||= []).push [f.lineno, arity.to_i]

Просмотреть файл

@ -61,7 +61,7 @@ def prelude(f, out)
return
when /\A%token/, /\A%type/, /\A} <node(?>_\w+)?>/
# types in %union which have corresponding set_yylval_* macro.
out << line.sub(/<(?:node(?>_\w+)?|num|id)>/, '<val>')
out << line
when /^enum lex_state_(?:bits|e) \{/
lex_state_def = true
out << line

Просмотреть файл

@ -126,7 +126,9 @@ struct MEMO {
rb_ary_set_len((value), offsetof(type, member) / sizeof(VALUE)), \
MEMO_FOR(type, value))
#ifndef RUBY_RUBYPARSER_H
typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t;
#endif
rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt);
struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc);
static inline enum imemo_type imemo_type(VALUE imemo);

Просмотреть файл

@ -67,9 +67,15 @@ int rb_ruby_parser_end_seen_p(rb_parser_t *p);
int rb_ruby_parser_set_yydebug(rb_parser_t *p, int flag);
rb_parser_string_t *rb_str_to_parser_string(rb_parser_t *p, VALUE str);
RUBY_SYMBOL_EXPORT_END
void rb_parser_warn_duplicate_keys(struct parser_params *p, NODE *hash);
int rb_parser_dvar_defined_ref(struct parser_params*, ID, ID**);
ID rb_parser_internal_id(struct parser_params*);
VALUE rb_parser_node_case_when_optimizable_literal(struct parser_params *p, const NODE *const node);
int rb_parser_reg_fragment_check(struct parser_params*, rb_parser_string_t*, int);
int rb_reg_named_capture_assign_iter_impl(struct parser_params *p, const char *s, long len, rb_encoding *enc, NODE **succ_block, const rb_code_location_t *loc);
int rb_parser_local_defined(struct parser_params *p, ID id, const struct rb_iseq_struct *iseq);
RUBY_SYMBOL_EXPORT_END
#ifdef RIPPER
void ripper_parser_mark(void *ptr);

Просмотреть файл

@ -18,6 +18,21 @@ rb_parser_t *rb_parser_params_new(void);
VALUE rb_parser_set_context(VALUE, const struct rb_iseq_struct *, int);
VALUE rb_parser_new(void);
rb_ast_t *rb_parser_compile_string_path(VALUE vparser, VALUE fname, VALUE src, int line);
VALUE rb_str_new_parser_string(rb_parser_string_t *str);
VALUE rb_node_str_string_val(const NODE *);
VALUE rb_node_sym_string_val(const NODE *);
VALUE rb_node_dstr_string_val(const NODE *);
VALUE rb_node_dregx_string_val(const NODE *);
VALUE rb_node_line_lineno_val(const NODE *);
VALUE rb_node_file_path_val(const NODE *);
VALUE rb_node_encoding_val(const NODE *);
VALUE rb_node_const_decl_val(const NODE *node);
VALUE rb_node_integer_literal_val(const NODE *);
VALUE rb_node_float_literal_val(const NODE *);
VALUE rb_node_rational_literal_val(const NODE *);
VALUE rb_node_imaginary_literal_val(const NODE *);
RUBY_SYMBOL_EXPORT_END
VALUE rb_parser_end_seen_p(VALUE);
@ -72,22 +87,4 @@ enum lex_state_e {
EXPR_NONE = 0
};
RUBY_SYMBOL_EXPORT_BEGIN
VALUE rb_str_new_parser_string(rb_parser_string_t *str);
RUBY_SYMBOL_EXPORT_END
VALUE rb_node_str_string_val(const NODE *);
VALUE rb_node_sym_string_val(const NODE *);
VALUE rb_node_dstr_string_val(const NODE *);
VALUE rb_node_dregx_string_val(const NODE *);
VALUE rb_node_line_lineno_val(const NODE *);
VALUE rb_node_file_path_val(const NODE *);
VALUE rb_node_encoding_val(const NODE *);
VALUE rb_node_const_decl_val(const NODE *node);
VALUE rb_node_integer_literal_val(const NODE *);
VALUE rb_node_float_literal_val(const NODE *);
VALUE rb_node_rational_literal_val(const NODE *);
VALUE rb_node_imaginary_literal_val(const NODE *);
#endif /* INTERNAL_RUBY_PARSE_H */

Просмотреть файл

@ -459,10 +459,6 @@ class RbInspector(LLDBInterface):
self._append_expression("*(struct RNode_LINE *) %0#x" % val.GetValueAsUnsigned())
elif nd_type == self.ruby_globals["NODE_FILE"]:
self._append_expression("*(struct RNode_FILE *) %0#x" % val.GetValueAsUnsigned())
elif nd_type == self.ruby_globals["NODE_RIPPER"]:
self._append_expression("*(struct RNode_RIPPER *) %0#x" % val.GetValueAsUnsigned())
elif nd_type == self.ruby_globals["NODE_RIPPER_VALUES"]:
self._append_expression("*(struct RNode_RIPPER_VALUES *) %0#x" % val.GetValueAsUnsigned())
else:
self._append_expression("*(struct RNode *) %0#x" % val.GetValueAsUnsigned())

19
node.c
Просмотреть файл

@ -69,7 +69,6 @@ rb_node_buffer_new(void)
init_node_buffer_list(&nb->unmarkable, (node_buffer_elem_t*)&nb[1], ruby_xmalloc);
init_node_buffer_list(&nb->markable, (node_buffer_elem_t*)((size_t)nb->unmarkable.head + bucket_size), ruby_xmalloc);
nb->local_tables = 0;
nb->mark_hash = Qnil;
nb->tokens = Qnil;
#ifdef UNIVERSAL_PARSER
nb->config = config;
@ -405,7 +404,6 @@ void
rb_ast_update_references(rb_ast_t *ast)
{
if (ast->node_buffer) {
ast->node_buffer->mark_hash = rb_gc_location(ast->node_buffer->mark_hash);
ast->node_buffer->tokens = rb_gc_location(ast->node_buffer->tokens);
node_buffer_t *nb = ast->node_buffer;
@ -419,7 +417,6 @@ void
rb_ast_mark(rb_ast_t *ast)
{
if (ast->node_buffer) {
rb_gc_mark_movable(ast->node_buffer->mark_hash);
rb_gc_mark_movable(ast->node_buffer->tokens);
node_buffer_t *nb = ast->node_buffer;
@ -470,22 +467,6 @@ rb_ast_dispose(rb_ast_t *ast)
rb_ast_free(ast);
}
void
rb_ast_add_mark_object(rb_ast_t *ast, VALUE obj)
{
if (NIL_P(ast->node_buffer->mark_hash)) {
RB_OBJ_WRITE(ast, &ast->node_buffer->mark_hash, rb_ident_hash_new());
}
rb_hash_aset(ast->node_buffer->mark_hash, obj, Qtrue);
}
void
rb_ast_delete_mark_object(rb_ast_t *ast, VALUE obj)
{
if (NIL_P(ast->node_buffer->mark_hash)) return;
rb_hash_delete(ast->node_buffer->mark_hash, obj);
}
VALUE
rb_ast_tokens(rb_ast_t *ast)
{

3
node.h
Просмотреть файл

@ -35,7 +35,6 @@ struct node_buffer_struct {
node_buffer_list_t unmarkable;
node_buffer_list_t markable;
struct rb_ast_local_table_link *local_tables;
VALUE mark_hash;
// - id (sequence number)
// - token_type
// - text of token
@ -66,8 +65,6 @@ void rb_node_init(NODE *n, enum node_type type);
void rb_ast_mark(rb_ast_t*);
void rb_ast_update_references(rb_ast_t*);
void rb_ast_free(rb_ast_t*);
void rb_ast_add_mark_object(rb_ast_t*, VALUE);
void rb_ast_delete_mark_object(rb_ast_t*, VALUE);
void rb_ast_set_tokens(rb_ast_t*, VALUE);
NODE *rb_ast_newnode(rb_ast_t*, enum node_type type, size_t size, size_t alignment);
void rb_ast_delete_node(rb_ast_t*, NODE *n);

Просмотреть файл

@ -1162,8 +1162,6 @@ dump_node(VALUE buf, VALUE indent, int comment, const NODE * node)
return;
case NODE_ARGS_AUX:
case NODE_RIPPER:
case NODE_RIPPER_VALUES:
case NODE_LAST:
break;
}

2463
parse.y

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@ -170,8 +170,6 @@ enum node_type {
NODE_LINE,
NODE_FILE,
NODE_ENCODING,
NODE_RIPPER,
NODE_RIPPER_VALUES,
NODE_LAST
};
@ -1138,27 +1136,6 @@ typedef struct RNode_ERROR {
#define RNODE_FILE(node) ((struct RNode_FILE *)(node))
#define RNODE_ENCODING(node) ((struct RNode_ENCODING *)(node))
#ifdef RIPPER
typedef struct RNode_RIPPER {
NODE node;
ID nd_vid;
VALUE nd_rval;
VALUE nd_cval;
} rb_node_ripper_t;
typedef struct RNode_RIPPER_VALUES {
NODE node;
VALUE nd_val1;
VALUE nd_val2;
VALUE nd_val3;
} rb_node_ripper_values_t;
#define RNODE_RIPPER(node) ((struct RNode_RIPPER *)(node))
#define RNODE_RIPPER_VALUES(node) ((struct RNode_RIPPER_VALUES *)(node))
#endif
/* FL : 0..4: T_TYPES, 5: KEEP_WB, 6: PROMOTED, 7: FINALIZE, 8: UNUSED, 9: UNUSED, 10: EXIVAR, 11: FREEZE */
/* NODE_FL: 0..4: T_TYPES, 5: KEEP_WB, 6: PROMOTED, 7: NODE_FL_NEWLINE,
* 8..14: nd_type,

Просмотреть файл

@ -88,7 +88,7 @@ class TestRipper::Ripper < Test::Unit::TestCase
ripper.yydebug = true
ripper.debug_output = out
ripper.parse
assert_include out.string[/.*"literal content".*/], 'woot'
assert_include out.string[/.*"literal content".*/], '1.1-1.5'
end
def test_regexp_with_option

Просмотреть файл

@ -18,6 +18,11 @@ class TestRipper::Sexp < Test::Unit::TestCase
assert_nil Ripper.sexp("/*")
assert_nil Ripper.sexp("/*/")
assert_nil Ripper.sexp("/+/")
assert_nil Ripper.sexp("m(&nil) {}"), '[Bug #10436]'
assert_nil Ripper.sexp("/(?<a>)/ =~ ''; x = a **a, **a if false"), '[Bug #18988]'
assert_nil Ripper.sexp("return + return"), '[Bug #20055]'
assert_nil Ripper.sexp("1 in [a, a]"), '[Bug #20055]'
assert_nil Ripper.sexp("1 + (1 => [a, a])"), '[Bug #20055]'
end
def test_regexp_content
@ -34,6 +39,14 @@ class TestRipper::Sexp < Test::Unit::TestCase
assert_equal '(?<n>a(b|\g<n>))', search_sexp(:@tstring_content, search_sexp(:regexp_literal, sexp))[1]
end
def test_regexp_named_capture
sexp = Ripper.sexp("/(?<a>)/ =~ ''; x = a **a, a if false")
assert_not_nil sexp, '[Bug #18988]'
sexp = Ripper.sexp("/(?<a>)/ =~ ''; a %(exit)")
assert_equal 'exit', search_sexp(:@ident, search_sexp(:paren, sexp))[1], '[Bug #18988]'
end
def test_heredoc_content
sexp = Ripper.sexp("<<E\nfoo\nE")
assert_equal "foo\n", search_sexp(:@tstring_content, sexp)[1]