ruby/node.h

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

515 строки
16 KiB
C
Исходник Обычный вид История

#ifndef RUBY_NODE_H
#define RUBY_NODE_H 1
/**********************************************************************
node.h -
$Author$
created at: Fri May 28 15:14:02 JST 1993
* encoding.c: provide basic features for M17N. * parse.y: encoding aware parsing. * parse.y (pragma_encoding): encoding specification pragma. * parse.y (rb_intern3): encoding specified symbols. * string.c (rb_str_length): length based on characters. for older behavior, bytesize method added. * string.c (rb_str_index_m): index based on characters. rindex as well. * string.c (succ_char): encoding aware succeeding string. * string.c (rb_str_reverse): reverse based on characters. * string.c (rb_str_inspect): encoding aware string description. * string.c (rb_str_upcase_bang): encoding aware case conversion. downcase, capitalize, swapcase as well. * string.c (rb_str_tr_bang): tr based on characters. delete, squeeze, tr_s, count as well. * string.c (rb_str_split_m): split based on characters. * string.c (rb_str_each_line): encoding aware each_line. * string.c (rb_str_each_char): added. iteration based on characters. * string.c (rb_str_strip_bang): encoding aware whitespace stripping. lstrip, rstrip as well. * string.c (rb_str_justify): encoding aware justifying (ljust, rjust, center). * string.c (str_encoding): get encoding attribute from a string. * re.c (rb_reg_initialize): encoding aware regular expression * sprintf.c (rb_str_format): formatting (i.e. length count) based on characters. * io.c (rb_io_getc): getc to return one-character string. for older behavior, getbyte method added. * ext/stringio/stringio.c (strio_getc): ditto. * io.c (rb_io_ungetc): allow pushing arbitrary string at the current reading point. * ext/stringio/stringio.c (strio_ungetc): ditto. * ext/strscan/strscan.c: encoding support. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@13261 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
2007-08-25 07:29:39 +04:00
Copyright (C) 1993-2007 Yukihiro Matsumoto
**********************************************************************/
#include "internal/compilers.h"
#if defined(__cplusplus)
extern "C" {
#if 0
} /* satisfy cc-mode */
#endif
#endif
enum node_type {
NODE_SCOPE,
NODE_BLOCK,
NODE_IF,
NODE_UNLESS,
NODE_CASE,
NODE_CASE2,
NODE_CASE3,
NODE_WHEN,
NODE_IN,
NODE_WHILE,
NODE_UNTIL,
NODE_ITER,
NODE_FOR,
NODE_FOR_MASGN,
NODE_BREAK,
NODE_NEXT,
NODE_REDO,
NODE_RETRY,
NODE_BEGIN,
NODE_RESCUE,
NODE_RESBODY,
NODE_ENSURE,
NODE_AND,
NODE_OR,
NODE_MASGN,
NODE_LASGN,
NODE_DASGN,
NODE_GASGN,
NODE_IASGN,
NODE_CDECL,
NODE_CVASGN,
NODE_OP_ASGN1,
NODE_OP_ASGN2,
NODE_OP_ASGN_AND,
NODE_OP_ASGN_OR,
NODE_OP_CDECL,
NODE_CALL,
NODE_OPCALL,
NODE_FCALL,
NODE_VCALL,
NODE_QCALL,
NODE_SUPER,
NODE_ZSUPER,
NODE_LIST,
NODE_ZLIST,
NODE_VALUES,
NODE_HASH,
NODE_RETURN,
NODE_YIELD,
NODE_LVAR,
NODE_DVAR,
NODE_GVAR,
NODE_IVAR,
NODE_CONST,
NODE_CVAR,
NODE_NTH_REF,
NODE_BACK_REF,
NODE_MATCH,
NODE_MATCH2,
NODE_MATCH3,
NODE_LIT,
NODE_STR,
NODE_DSTR,
NODE_XSTR,
NODE_DXSTR,
NODE_EVSTR,
NODE_DREGX,
NODE_ONCE,
NODE_ARGS,
NODE_ARGS_AUX,
NODE_OPT_ARG,
NODE_KW_ARG,
* sprintf.c (rb_str_format): allow %c to print one character string (e.g. ?x). * lib/tempfile.rb (Tempfile::make_tmpname): put dot between basename and pid. [ruby-talk:196272] * parse.y (do_block): remove -> style block. * parse.y (parser_yylex): remove tLAMBDA_ARG. * eval.c (rb_call0): binding for the return event hook should have consistent scope. [ruby-core:07928] * eval.c (proc_invoke): return behavior should depend whether it is surrounded by a lambda or a mere block. * eval.c (formal_assign): handles post splat arguments. * eval.c (rb_call0): ditto. * st.c (strhash): use FNV-1a hash. * parse.y (parser_yylex): removed experimental ';;' terminator. * eval.c (rb_node_arity): should be aware of post splat arguments. * eval.c (rb_proc_arity): ditto. * parse.y (f_args): syntax rule enhanced to support arguments after the splat. * parse.y (block_param): ditto for block parameters. * parse.y (f_post_arg): mandatory formal arguments after the splat argument. * parse.y (new_args_gen): generate nodes for mandatory formal arguments after the splat argument. * eval.c (rb_eval): dispatch mandatory formal arguments after the splat argument. * parse.y (args): allow more than one splat in the argument list. * parse.y (method_call): allow aref [] to accept all kind of method argument, including assocs, splat, and block argument. * eval.c (SETUP_ARGS0): prepare block argument as well. * lib/mathn.rb (Integer): remove Integer#gcd2. [ruby-core:07931] * eval.c (error_line): print receivers true/false/nil specially. * eval.c (rb_proc_yield): handles parameters in yield semantics. * eval.c (nil_yield): gives LocalJumpError to denote no block error. * io.c (rb_io_getc): now takes one-character string. * string.c (rb_str_hash): use FNV-1a hash from Fowler/Noll/Vo hashing algorithm. * string.c (rb_str_aref): str[0] now returns 1 character string, instead of a fixnum. [Ruby2] * parse.y (parser_yylex): ?c now returns 1 character string, instead of a fixnum. [Ruby2] * string.c (rb_str_aset): no longer support fixnum insertion. * eval.c (umethod_bind): should not update original class. [ruby-dev:28636] * eval.c (ev_const_get): should support constant access from within instance_eval(). [ruby-dev:28327] * time.c (time_timeval): should round for usec floating number. [ruby-core:07896] * time.c (time_add): ditto. * dir.c (sys_warning): should not call a vararg function rb_sys_warning() indirectly. [ruby-core:07886] * numeric.c (flo_divmod): the first element of Float#divmod should be an integer. [ruby-dev:28589] * test/ruby/test_float.rb: add tests for divmod, div, modulo and remainder. * re.c (rb_reg_initialize): should not allow modifying literal regexps. frozen check moved from rb_reg_initialize_m as well. * re.c (rb_reg_initialize): should not modify untainted objects in safe levels higher than 3. * re.c (rb_memcmp): type change from char* to const void*. * dir.c (dir_close): should not close untainted dir stream. * dir.c (GetDIR): add tainted/frozen check for each dir operation. * lib/rdoc/parsers/parse_rb.rb (RDoc::RubyParser::parse_symbol_arg): typo fixed. a patch from Florian Gross <florg at florg.net>. * eval.c (EXEC_EVENT_HOOK): trace_func may remove itself from event_hooks. no guarantee for arbitrary hook deletion. [ruby-dev:28632] * util.c (ruby_strtod): differ addition to minimize error. [ruby-dev:28619] * util.c (ruby_strtod): should not raise ERANGE when the input string does not have any digits. [ruby-dev:28629] * eval.c (proc_invoke): should restore old ruby_frame->block. thanks to ts <decoux at moulon.inra.fr>. [ruby-core:07833] also fix [ruby-dev:28614] as well. * signal.c (trap): sig should be less then NSIG. Coverity found this bug. a patch from Kevin Tew <tewk at tewk.com>. [ruby-core:07823] * math.c (math_log2): add new method inspired by [ruby-talk:191237]. * math.c (math_log): add optional base argument to Math::log(). [ruby-talk:191308] * ext/syck/emitter.c (syck_scan_scalar): avoid accessing uninitialized array element. a patch from Pat Eyler <rubypate at gmail.com>. [ruby-core:07809] * array.c (rb_ary_fill): initialize local variables first. a patch from Pat Eyler <rubypate at gmail.com>. [ruby-core:07810] * ext/syck/yaml2byte.c (syck_yaml2byte_handler): need to free type_tag. a patch from Pat Eyler <rubypate at gmail.com>. [ruby-core:07808] * ext/socket/socket.c (make_hostent_internal): accept ai_family check from Sam Roberts <sroberts at uniserve.com>. [ruby-core:07691] * util.c (ruby_strtod): should not cut off 18 digits for no reason. [ruby-core:07796] * array.c (rb_ary_fill): internalize local variable "beg" to pacify Coverity. [ruby-core:07770] * pack.c (pack_unpack): now supports CRLF newlines. a patch from <tommy at tmtm.org>. [ruby-dev:28601] * applied code clean-up patch from Stefan Huehner <stefan at huehner.org>. [ruby-core:07764] * lib/jcode.rb (String::tr_s): should have translated non squeezing character sequence (i.e. a character) as well. thanks to Hiroshi Ichikawa <gimite at gimite.ddo.jp> [ruby-list:42090] * ext/socket/socket.c: document update patch from Sam Roberts <sroberts at uniserve.com>. [ruby-core:07701] * lib/mathn.rb (Integer): need not to remove gcd2. a patch from NARUSE, Yui <naruse at airemix.com>. [ruby-dev:28570] * parse.y (arg): too much NEW_LIST() * eval.c (SETUP_ARGS0): remove unnecessary access to nd_alen. * eval.c (rb_eval): use ARGSCAT for NODE_OP_ASGN1. [ruby-dev:28585] * parse.y (arg): use NODE_ARGSCAT for placeholder. * lib/getoptlong.rb (GetoptLong::get): RDoc update patch from mathew <meta at pobox.com>. [ruby-core:07738] * variable.c (rb_const_set): raise error when no target klass is supplied. [ruby-dev:28582] * prec.c (prec_prec_f): documentation patch from <gerardo.santana at gmail.com>. [ruby-core:07689] * bignum.c (rb_big_pow): second operand may be too big even if it's a Fixnum. [ruby-talk:187984] * README.EXT: update symbol description. [ruby-talk:188104] * COPYING: explicitly note GPLv2. [ruby-talk:187922] * parse.y: remove some obsolete syntax rules (unparenthesized method calls in argument list). * eval.c (rb_call0): insecure calling should be checked for non NODE_SCOPE method invocations too. * eval.c (rb_alias): should preserve the current safe level as well as method definition. * process.c (rb_f_sleep): remove RDoc description about SIGALRM which is not valid on the current implementation. [ruby-dev:28464] Thu Mar 23 21:40:47 2006 K.Kosako <sndgk393 AT ybb.ne.jp> * eval.c (method_missing): should support argument splat in super. a bug in combination of super, splat and method_missing. [ruby-talk:185438] * configure.in: Solaris SunPro compiler -rapth patch from <kuwa at labs.fujitsu.com>. [ruby-dev:28443] * configure.in: remove enable_rpath=no for Solaris. [ruby-dev:28440] * ext/win32ole/win32ole.c (ole_val2olevariantdata): change behavior of converting OLE Variant object with VT_ARRAY|VT_UI1 and Ruby String object. * ruby.1: a clarification patch from David Lutterkort <dlutter at redhat.com>. [ruby-core:7508] * lib/rdoc/ri/ri_paths.rb (RI::Paths): adding paths from rubygems directories. a patch from Eric Hodel <drbrain at segment7.net>. [ruby-core:07423] * eval.c (rb_clear_cache_by_class): clearing wrong cache. * ext/extmk.rb: use :remove_destination to install extension libraries to avoid SEGV. [ruby-dev:28417] * eval.c (rb_thread_fd_writable): should not re-schedule output from KILLED thread (must be error printing). * array.c (rb_ary_flatten_bang): allow specifying recursion level. [ruby-talk:182170] * array.c (rb_ary_flatten): ditto. * gc.c (add_heap): a heap_slots may overflow. a patch from Stefan Weil <weil at mail.berlios.de>. * eval.c (rb_call): use separate cache for fcall/vcall invocation. * eval.c (rb_eval): NODE_FCALL, NODE_VCALL can call local functions. * eval.c (rb_mod_local): a new method to specify newly added visibility "local". * eval.c (search_method): search for local methods which are visible only from the current class. * class.c (rb_class_local_methods): a method to list local methods. * object.c (Init_Object): add BasicObject class as a top level BlankSlate class. * ruby.h (SYM2ID): should not cast to signed long. [ruby-core:07414] * class.c (rb_include_module): allow module duplication. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@10235 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
2006-06-10 01:20:17 +04:00
NODE_POSTARG,
NODE_ARGSCAT,
NODE_ARGSPUSH,
NODE_SPLAT,
NODE_BLOCK_PASS,
NODE_DEFN,
NODE_DEFS,
NODE_ALIAS,
NODE_VALIAS,
NODE_UNDEF,
NODE_CLASS,
NODE_MODULE,
NODE_SCLASS,
NODE_COLON2,
NODE_COLON3,
NODE_DOT2,
NODE_DOT3,
NODE_FLIP2,
NODE_FLIP3,
NODE_SELF,
NODE_NIL,
NODE_TRUE,
NODE_FALSE,
NODE_ERRINFO,
NODE_DEFINED,
NODE_POSTEXE,
NODE_DSYM,
NODE_ATTRASGN,
NODE_LAMBDA,
NODE_ARYPTN,
NODE_HSHPTN,
NODE_FNDPTN,
NODE_ERROR,
NODE_LAST
};
typedef struct rb_code_position_struct {
int lineno;
int column;
} rb_code_position_t;
typedef struct rb_code_location_struct {
rb_code_position_t beg_pos;
rb_code_position_t end_pos;
} rb_code_location_t;
2019-07-19 00:15:47 +03:00
static inline rb_code_location_t
2020-05-14 11:15:03 +03:00
code_loc_gen(const rb_code_location_t *loc1, const rb_code_location_t *loc2)
{
rb_code_location_t loc;
loc.beg_pos = loc1->beg_pos;
loc.end_pos = loc2->end_pos;
return loc;
}
typedef struct rb_ast_id_table {
int size;
ID ids[FLEX_ARY_LEN];
} rb_ast_id_table_t;
typedef struct RNode {
VALUE flags;
union {
struct RNode *node;
ID id;
VALUE value;
rb_ast_id_table_t *tbl;
} u1;
union {
struct RNode *node;
ID id;
long argc;
VALUE value;
} u2;
union {
struct RNode *node;
ID id;
long state;
struct rb_args_info *args;
2020-05-04 07:24:53 +03:00
struct rb_ary_pattern_info *apinfo;
struct rb_fnd_pattern_info *fpinfo;
VALUE value;
} u3;
rb_code_location_t nd_loc;
int node_id;
} NODE;
#define RNODE(obj) ((struct RNode *)(obj))
/* FL : 0..4: T_TYPES, 5: KEEP_WB, 6: PROMOTED, 7: FINALIZE, 8: UNUSED, 9: UNUSED, 10: EXIVAR, 11: FREEZE */
/* NODE_FL: 0..4: T_TYPES, 5: KEEP_WB, 6: PROMOTED, 7: NODE_FL_NEWLINE,
* gc.c: support RGENGC. [ruby-trunk - Feature #8339] See this ticet about RGENGC. * gc.c: Add several flags: * RGENGC_DEBUG: if >0, then prints debug information. * RGENGC_CHECK_MODE: if >0, add assertions. * RGENGC_PROFILE: if >0, add profiling features. check GC.stat and GC::Profiler. * include/ruby/ruby.h: disable RGENGC by default (USE_RGENGC == 0). * array.c: add write barriers for T_ARRAY and generate sunny objects. * include/ruby/ruby.h (RARRAY_PTR_USE): added. Use this macro if you want to access raw pointers. If you modify the contents which pointer pointed, then you need to care write barrier. * bignum.c, marshal.c, random.c: generate T_BIGNUM sunny objects. * complex.c, include/ruby/ruby.h: add write barriers for T_COMPLEX and generate sunny objects. * rational.c (nurat_s_new_internal), include/ruby/ruby.h: add write barriers for T_RATIONAL and generate sunny objects. * internal.h: add write barriers for RBasic::klass. * numeric.c (rb_float_new_in_heap): generate sunny T_FLOAT objects. * object.c (rb_class_allocate_instance), range.c: generate sunny T_OBJECT objects. * string.c: add write barriers for T_STRING and generate sunny objects. * variable.c: add write barriers for ivars. * vm_insnhelper.c (vm_setivar): ditto. * include/ruby/ruby.h, debug.c: use two flags FL_WB_PROTECTED and FL_OLDGEN. * node.h (NODE_FL_CREF_PUSHED_BY_EVAL, NODE_FL_CREF_OMOD_SHARED): move flag bits. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@40703 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
2013-05-13 22:07:47 +04:00
* 8..14: nd_type,
* 15..: nd_line
* gc.c: support RGENGC. [ruby-trunk - Feature #8339] See this ticet about RGENGC. * gc.c: Add several flags: * RGENGC_DEBUG: if >0, then prints debug information. * RGENGC_CHECK_MODE: if >0, add assertions. * RGENGC_PROFILE: if >0, add profiling features. check GC.stat and GC::Profiler. * include/ruby/ruby.h: disable RGENGC by default (USE_RGENGC == 0). * array.c: add write barriers for T_ARRAY and generate sunny objects. * include/ruby/ruby.h (RARRAY_PTR_USE): added. Use this macro if you want to access raw pointers. If you modify the contents which pointer pointed, then you need to care write barrier. * bignum.c, marshal.c, random.c: generate T_BIGNUM sunny objects. * complex.c, include/ruby/ruby.h: add write barriers for T_COMPLEX and generate sunny objects. * rational.c (nurat_s_new_internal), include/ruby/ruby.h: add write barriers for T_RATIONAL and generate sunny objects. * internal.h: add write barriers for RBasic::klass. * numeric.c (rb_float_new_in_heap): generate sunny T_FLOAT objects. * object.c (rb_class_allocate_instance), range.c: generate sunny T_OBJECT objects. * string.c: add write barriers for T_STRING and generate sunny objects. * variable.c: add write barriers for ivars. * vm_insnhelper.c (vm_setivar): ditto. * include/ruby/ruby.h, debug.c: use two flags FL_WB_PROTECTED and FL_OLDGEN. * node.h (NODE_FL_CREF_PUSHED_BY_EVAL, NODE_FL_CREF_OMOD_SHARED): move flag bits. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@40703 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
2013-05-13 22:07:47 +04:00
*/
#define NODE_FL_NEWLINE (((VALUE)1)<<7)
#define NODE_TYPESHIFT 8
#define NODE_TYPEMASK (((VALUE)0x7f)<<NODE_TYPESHIFT)
#define nd_type(n) ((int) (((n)->flags & NODE_TYPEMASK)>>NODE_TYPESHIFT))
#define nd_set_type(n,t) \
rb_node_set_type(n, t)
#define nd_init_type(n,t) \
(n)->flags=(((n)->flags&~NODE_TYPEMASK)|((((unsigned long)(t))<<NODE_TYPESHIFT)&NODE_TYPEMASK))
#define NODE_LSHIFT (NODE_TYPESHIFT+7)
#define NODE_LMASK (((SIGNED_VALUE)1<<(sizeof(VALUE)*CHAR_BIT-NODE_LSHIFT))-1)
#define nd_line(n) (int)(((SIGNED_VALUE)(n)->flags)>>NODE_LSHIFT)
#define nd_set_line(n,l) \
(n)->flags=(((n)->flags&~((VALUE)(-1)<<NODE_LSHIFT))|((VALUE)((l)&NODE_LMASK)<<NODE_LSHIFT))
#define nd_first_column(n) ((int)((n)->nd_loc.beg_pos.column))
#define nd_set_first_column(n, v) ((n)->nd_loc.beg_pos.column = (v))
#define nd_first_lineno(n) ((int)((n)->nd_loc.beg_pos.lineno))
#define nd_set_first_lineno(n, v) ((n)->nd_loc.beg_pos.lineno = (v))
#define nd_first_loc(n) ((n)->nd_loc.beg_pos)
#define nd_set_first_loc(n, v) (nd_first_loc(n) = (v))
#define nd_last_column(n) ((int)((n)->nd_loc.end_pos.column))
#define nd_set_last_column(n, v) ((n)->nd_loc.end_pos.column = (v))
#define nd_last_lineno(n) ((int)((n)->nd_loc.end_pos.lineno))
#define nd_set_last_lineno(n, v) ((n)->nd_loc.end_pos.lineno = (v))
#define nd_last_loc(n) ((n)->nd_loc.end_pos)
#define nd_set_last_loc(n, v) (nd_last_loc(n) = (v))
#define nd_node_id(n) ((n)->node_id)
#define nd_set_node_id(n,id) ((n)->node_id = (id))
#define nd_head u1.node
#define nd_alen u2.argc
#define nd_next u3.node
#define nd_cond u1.node
#define nd_body u2.node
#define nd_else u3.node
#define nd_resq u2.node
#define nd_ensr u3.node
#define nd_1st u1.node
#define nd_2nd u2.node
#define nd_stts u1.node
#define nd_entry u3.id
#define nd_vid u1.id
#define nd_var u1.node
#define nd_iter u3.node
#define nd_value u2.node
#define nd_aid u3.id
#define nd_lit u1.value
#define nd_recv u1.node
#define nd_mid u2.id
#define nd_args u3.node
#define nd_ainfo u3.args
#define nd_defn u3.node
#define nd_cpath u1.node
#define nd_super u3.node
#define nd_beg u1.node
#define nd_end u2.node
#define nd_state u3.state
#define nd_nth u2.argc
#define nd_alias u1.id
#define nd_orig u2.id
#define nd_undef u2.node
#define nd_brace u2.argc
#define nd_pconst u1.node
#define nd_pkwargs u2.node
#define nd_pkwrestarg u3.node
#define nd_apinfo u3.apinfo
#define nd_fpinfo u3.fpinfo
// for NODE_SCOPE
#define nd_tbl u1.tbl
// for NODE_ARGS_AUX
#define nd_pid u1.id
#define nd_plen u2.argc
#define nd_cflag u2.id
// for ripper
#define nd_cval u3.value
#define nd_rval u2.value
#define nd_tag u1.id
#define NEW_NODE(t,a0,a1,a2,loc) rb_node_newnode((t),(VALUE)(a0),(VALUE)(a1),(VALUE)(a2),loc)
#define NEW_NODE_WITH_LOCALS(t,a1,a2,loc) node_newnode_with_locals(p, (t),(VALUE)(a1),(VALUE)(a2),loc)
#define NEW_DEFN(i,a,d,loc) NEW_NODE(NODE_DEFN,0,i,NEW_SCOPE(a,d,loc),loc)
#define NEW_DEFS(r,i,a,d,loc) NEW_NODE(NODE_DEFS,r,i,NEW_SCOPE(a,d,loc),loc)
#define NEW_SCOPE(a,b,loc) NEW_NODE_WITH_LOCALS(NODE_SCOPE,b,a,loc)
#define NEW_BLOCK(a,loc) NEW_NODE(NODE_BLOCK,a,0,0,loc)
#define NEW_IF(c,t,e,loc) NEW_NODE(NODE_IF,c,t,e,loc)
#define NEW_UNLESS(c,t,e,loc) NEW_NODE(NODE_UNLESS,c,t,e,loc)
#define NEW_CASE(h,b,loc) NEW_NODE(NODE_CASE,h,b,0,loc)
#define NEW_CASE2(b,loc) NEW_NODE(NODE_CASE2,0,b,0,loc)
#define NEW_CASE3(h,b,loc) NEW_NODE(NODE_CASE3,h,b,0,loc)
#define NEW_WHEN(c,t,e,loc) NEW_NODE(NODE_WHEN,c,t,e,loc)
#define NEW_IN(c,t,e,loc) NEW_NODE(NODE_IN,c,t,e,loc)
#define NEW_WHILE(c,b,n,loc) NEW_NODE(NODE_WHILE,c,b,n,loc)
#define NEW_UNTIL(c,b,n,loc) NEW_NODE(NODE_UNTIL,c,b,n,loc)
#define NEW_FOR(i,b,loc) NEW_NODE(NODE_FOR,0,b,i,loc)
#define NEW_FOR_MASGN(v,loc) NEW_NODE(NODE_FOR_MASGN,v,0,0,loc)
#define NEW_ITER(a,b,loc) NEW_NODE(NODE_ITER,0,NEW_SCOPE(a,b,loc),0,loc)
#define NEW_LAMBDA(a,b,loc) NEW_NODE(NODE_LAMBDA,0,NEW_SCOPE(a,b,loc),0,loc)
#define NEW_BREAK(s,loc) NEW_NODE(NODE_BREAK,s,0,0,loc)
#define NEW_NEXT(s,loc) NEW_NODE(NODE_NEXT,s,0,0,loc)
#define NEW_REDO(loc) NEW_NODE(NODE_REDO,0,0,0,loc)
#define NEW_RETRY(loc) NEW_NODE(NODE_RETRY,0,0,0,loc)
#define NEW_BEGIN(b,loc) NEW_NODE(NODE_BEGIN,0,b,0,loc)
#define NEW_RESCUE(b,res,e,loc) NEW_NODE(NODE_RESCUE,b,res,e,loc)
#define NEW_RESBODY(a,ex,n,loc) NEW_NODE(NODE_RESBODY,n,ex,a,loc)
#define NEW_ENSURE(b,en,loc) NEW_NODE(NODE_ENSURE,b,0,en,loc)
#define NEW_RETURN(s,loc) NEW_NODE(NODE_RETURN,s,0,0,loc)
#define NEW_YIELD(a,loc) NEW_NODE(NODE_YIELD,a,0,0,loc)
#define NEW_LIST(a,loc) NEW_NODE(NODE_LIST,a,1,0,loc)
#define NEW_ZLIST(loc) NEW_NODE(NODE_ZLIST,0,0,0,loc)
#define NEW_HASH(a,loc) NEW_NODE(NODE_HASH,a,0,0,loc)
#define NEW_MASGN(l,r,loc) NEW_NODE(NODE_MASGN,l,0,r,loc)
#define NEW_GASGN(v,val,loc) NEW_NODE(NODE_GASGN,v,val,v,loc)
#define NEW_LASGN(v,val,loc) NEW_NODE(NODE_LASGN,v,val,0,loc)
#define NEW_DASGN(v,val,loc) NEW_NODE(NODE_DASGN,v,val,0,loc)
#define NEW_IASGN(v,val,loc) NEW_NODE(NODE_IASGN,v,val,0,loc)
#define NEW_CDECL(v,val,path,loc) NEW_NODE(NODE_CDECL,v,val,path,loc)
#define NEW_CVASGN(v,val,loc) NEW_NODE(NODE_CVASGN,v,val,0,loc)
#define NEW_OP_ASGN1(p,id,a,loc) NEW_NODE(NODE_OP_ASGN1,p,id,a,loc)
#define NEW_OP_ASGN2(r,t,i,o,val,loc) NEW_NODE(NODE_OP_ASGN2,r,val,NEW_OP_ASGN22(i,o,t,loc),loc)
#define NEW_OP_ASGN22(i,o,t,loc) NEW_NODE(NODE_OP_ASGN2,i,o,t,loc)
#define NEW_OP_ASGN_OR(i,val,loc) NEW_NODE(NODE_OP_ASGN_OR,i,val,0,loc)
#define NEW_OP_ASGN_AND(i,val,loc) NEW_NODE(NODE_OP_ASGN_AND,i,val,0,loc)
#define NEW_OP_CDECL(v,op,val,loc) NEW_NODE(NODE_OP_CDECL,v,val,op,loc)
#define NEW_GVAR(v,loc) NEW_NODE(NODE_GVAR,v,0,v,loc)
#define NEW_LVAR(v,loc) NEW_NODE(NODE_LVAR,v,0,0,loc)
#define NEW_DVAR(v,loc) NEW_NODE(NODE_DVAR,v,0,0,loc)
#define NEW_IVAR(v,loc) NEW_NODE(NODE_IVAR,v,0,0,loc)
#define NEW_CONST(v,loc) NEW_NODE(NODE_CONST,v,0,0,loc)
#define NEW_CVAR(v,loc) NEW_NODE(NODE_CVAR,v,0,0,loc)
#define NEW_NTH_REF(n,loc) NEW_NODE(NODE_NTH_REF,0,n,0,loc)
#define NEW_BACK_REF(n,loc) NEW_NODE(NODE_BACK_REF,0,n,0,loc)
#define NEW_MATCH(c,loc) NEW_NODE(NODE_MATCH,c,0,0,loc)
#define NEW_MATCH2(n1,n2,loc) NEW_NODE(NODE_MATCH2,n1,n2,0,loc)
#define NEW_MATCH3(r,n2,loc) NEW_NODE(NODE_MATCH3,r,n2,0,loc)
#define NEW_LIT(l,loc) NEW_NODE(NODE_LIT,l,0,0,loc)
#define NEW_STR(s,loc) NEW_NODE(NODE_STR,s,0,0,loc)
#define NEW_DSTR(s,loc) NEW_NODE(NODE_DSTR,s,1,0,loc)
#define NEW_XSTR(s,loc) NEW_NODE(NODE_XSTR,s,0,0,loc)
#define NEW_DXSTR(s,loc) NEW_NODE(NODE_DXSTR,s,0,0,loc)
#define NEW_DSYM(s,loc) NEW_NODE(NODE_DSYM,s,0,0,loc)
#define NEW_EVSTR(n,loc) NEW_NODE(NODE_EVSTR,0,(n),0,loc)
#define NEW_CALL(r,m,a,loc) NEW_NODE(NODE_CALL,r,m,a,loc)
#define NEW_OPCALL(r,m,a,loc) NEW_NODE(NODE_OPCALL,r,m,a,loc)
#define NEW_FCALL(m,a,loc) NEW_NODE(NODE_FCALL,0,m,a,loc)
#define NEW_VCALL(m,loc) NEW_NODE(NODE_VCALL,0,m,0,loc)
#define NEW_SUPER(a,loc) NEW_NODE(NODE_SUPER,0,0,a,loc)
#define NEW_ZSUPER(loc) NEW_NODE(NODE_ZSUPER,0,0,0,loc)
#define NEW_ARGS_AUX(r,b,loc) NEW_NODE(NODE_ARGS_AUX,r,b,0,loc)
#define NEW_OPT_ARG(i,v,loc) NEW_NODE(NODE_OPT_ARG,i,v,0,loc)
#define NEW_KW_ARG(i,v,loc) NEW_NODE(NODE_KW_ARG,i,v,0,loc)
#define NEW_POSTARG(i,v,loc) NEW_NODE(NODE_POSTARG,i,v,0,loc)
#define NEW_ARGSCAT(a,b,loc) NEW_NODE(NODE_ARGSCAT,a,b,0,loc)
#define NEW_ARGSPUSH(a,b,loc) NEW_NODE(NODE_ARGSPUSH,a,b,0,loc)
#define NEW_SPLAT(a,loc) NEW_NODE(NODE_SPLAT,a,0,0,loc)
#define NEW_BLOCK_PASS(b,loc) NEW_NODE(NODE_BLOCK_PASS,0,b,0,loc)
#define NEW_ALIAS(n,o,loc) NEW_NODE(NODE_ALIAS,n,o,0,loc)
#define NEW_VALIAS(n,o,loc) NEW_NODE(NODE_VALIAS,n,o,0,loc)
#define NEW_UNDEF(i,loc) NEW_NODE(NODE_UNDEF,0,i,0,loc)
#define NEW_CLASS(n,b,s,loc) NEW_NODE(NODE_CLASS,n,NEW_SCOPE(0,b,loc),(s),loc)
#define NEW_SCLASS(r,b,loc) NEW_NODE(NODE_SCLASS,r,NEW_SCOPE(0,b,loc),0,loc)
#define NEW_MODULE(n,b,loc) NEW_NODE(NODE_MODULE,n,NEW_SCOPE(0,b,loc),0,loc)
#define NEW_COLON2(c,i,loc) NEW_NODE(NODE_COLON2,c,i,0,loc)
#define NEW_COLON3(i,loc) NEW_NODE(NODE_COLON3,0,i,0,loc)
#define NEW_DOT2(b,e,loc) NEW_NODE(NODE_DOT2,b,e,0,loc)
#define NEW_DOT3(b,e,loc) NEW_NODE(NODE_DOT3,b,e,0,loc)
#define NEW_SELF(loc) NEW_NODE(NODE_SELF,0,0,1,loc)
#define NEW_NIL(loc) NEW_NODE(NODE_NIL,0,0,0,loc)
#define NEW_TRUE(loc) NEW_NODE(NODE_TRUE,0,0,0,loc)
#define NEW_FALSE(loc) NEW_NODE(NODE_FALSE,0,0,0,loc)
#define NEW_ERRINFO(loc) NEW_NODE(NODE_ERRINFO,0,0,0,loc)
#define NEW_DEFINED(e,loc) NEW_NODE(NODE_DEFINED,e,0,0,loc)
#define NEW_POSTEXE(b,loc) NEW_NODE(NODE_POSTEXE,0,b,0,loc)
#define NEW_ATTRASGN(r,m,a,loc) NEW_NODE(NODE_ATTRASGN,r,m,a,loc)
#define NEW_ERROR(loc) NEW_NODE(NODE_ERROR,0,0,0,loc)
#define NODE_SPECIAL_REQUIRED_KEYWORD ((NODE *)-1)
#define NODE_REQUIRED_KEYWORD_P(node) ((node)->nd_value == NODE_SPECIAL_REQUIRED_KEYWORD)
#define NODE_SPECIAL_NO_NAME_REST ((NODE *)-1)
#define NODE_NAMED_REST_P(node) ((node) != NODE_SPECIAL_NO_NAME_REST)
#define NODE_SPECIAL_EXCESSIVE_COMMA ((ID)1)
#define NODE_SPECIAL_NO_REST_KEYWORD ((NODE *)-1)
VALUE rb_node_case_when_optimizable_literal(const NODE *const node);
RUBY_SYMBOL_EXPORT_BEGIN
typedef struct node_buffer_struct node_buffer_t;
/* T_IMEMO/ast */
typedef struct rb_ast_body_struct {
const NODE *root;
VALUE compile_option;
VALUE script_lines;
// script_lines is either:
// - a Fixnum that represents the line count of the original source, or
// - an Array that contains the lines of the original source
} rb_ast_body_t;
typedef struct rb_ast_struct {
VALUE flags;
node_buffer_t *node_buffer;
rb_ast_body_t body;
} rb_ast_t;
rb_ast_t *rb_ast_new(void);
void rb_ast_mark(rb_ast_t*);
void rb_ast_update_references(rb_ast_t*);
void rb_ast_dispose(rb_ast_t*);
void rb_ast_free(rb_ast_t*);
size_t rb_ast_memsize(const rb_ast_t*);
void rb_ast_add_mark_object(rb_ast_t*, VALUE);
Enhance keep_tokens option for RubyVM::AbstractSyntaxTree parsing methods Implementation for Language Server Protocol (LSP) sometimes needs token information. For example both `m(1)` and `m(1, )` has same AST structure other than node locations then it's impossible to check the existence of `,` from AST. However in later case, it might be better to suggest variables list for the second argument. Token information is important for such case. This commit adds these methods. * Add `keep_tokens` option for `RubyVM::AbstractSyntaxTree.parse`, `.parse_file` and `.of` * Add `RubyVM::AbstractSyntaxTree::Node#tokens` which returns tokens for the node including tokens for descendants nodes. * Add `RubyVM::AbstractSyntaxTree::Node#all_tokens` which returns all tokens for the input script regardless the receiver node. [Feature #19070] Impacts on memory usage and performance are below: Memory usage: ``` $ cat test.rb root = RubyVM::AbstractSyntaxTree.parse_file(File.expand_path('../test/ruby/test_keyword.rb', __FILE__), keep_tokens: true) $ /usr/bin/time -f %Mkb /usr/local/bin/ruby -v ruby 3.2.0dev (2022-11-19T09:41:54Z 19070-keep_tokens d3af1b8057) [x86_64-linux] 11408kb # keep_tokens :false $ /usr/bin/time -f %Mkb /usr/local/bin/ruby test.rb 17508kb # keep_tokens :true $ /usr/bin/time -f %Mkb /usr/local/bin/ruby test.rb 30960kb ``` Performance: ``` $ cat ../ast_keep_tokens.yml prelude: | src = <<~SRC module M class C def m1(a, b) 1 + a + b end end end SRC benchmark: without_keep_tokens: | RubyVM::AbstractSyntaxTree.parse(src, keep_tokens: false) with_keep_tokens: | RubyVM::AbstractSyntaxTree.parse(src, keep_tokens: true) $ make benchmark COMPARE_RUBY="./ruby" ARGS=../ast_keep_tokens.yml /home/kaneko.y/.rbenv/shims/ruby --disable=gems -rrubygems -I../benchmark/lib ../benchmark/benchmark-driver/exe/benchmark-driver \ --executables="compare-ruby::./ruby -I.ext/common --disable-gem" \ --executables="built-ruby::./miniruby -I../lib -I. -I.ext/common ../tool/runruby.rb --extout=.ext -- --disable-gems --disable-gem" \ --output=markdown --output-compare -v ../ast_keep_tokens.yml compare-ruby: ruby 3.2.0dev (2022-11-19T09:41:54Z 19070-keep_tokens d3af1b8057) [x86_64-linux] built-ruby: ruby 3.2.0dev (2022-11-19T09:41:54Z 19070-keep_tokens d3af1b8057) [x86_64-linux] warming up.. | |compare-ruby|built-ruby| |:--------------------|-----------:|---------:| |without_keep_tokens | 21.659k| 21.303k| | | 1.02x| -| |with_keep_tokens | 6.220k| 5.691k| | | 1.09x| -| ```
2022-09-23 16:40:02 +03:00
void rb_ast_set_tokens(rb_ast_t*, VALUE);
VALUE rb_ast_tokens(rb_ast_t *ast);
NODE *rb_ast_newnode(rb_ast_t*, enum node_type type);
void rb_ast_delete_node(rb_ast_t*, NODE *n);
rb_ast_id_table_t *rb_ast_new_local_table(rb_ast_t*, int);
rb_ast_id_table_t *rb_ast_resize_latest_local_table(rb_ast_t*, int);
VALUE rb_parser_new(void);
VALUE rb_parser_end_seen_p(VALUE);
VALUE rb_parser_encoding(VALUE);
VALUE rb_parser_set_yydebug(VALUE, VALUE);
VALUE rb_parser_dump_tree(const NODE *node, int comment);
void rb_parser_set_options(VALUE, int, int, int, int);
rb_ast_t *rb_parser_compile_string(VALUE, const char*, VALUE, int);
rb_ast_t *rb_parser_compile_string_path(VALUE vparser, VALUE fname, VALUE src, int line);
rb_ast_t *rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE input, int line);
rb_ast_t *rb_parser_compile_generic(VALUE vparser, VALUE (*lex_gets)(VALUE, int), VALUE fname, VALUE input, int line);
void rb_node_init(NODE *n, enum node_type type, VALUE a0, VALUE a1, VALUE a2);
const char *ruby_node_name(int node);
const struct kwtable *rb_reserved_word(const char *, unsigned int);
struct rb_args_info {
NODE *pre_init;
NODE *post_init;
int pre_args_num; /* count of mandatory pre-arguments */
int post_args_num; /* count of mandatory post-arguments */
ID first_post_arg;
ID rest_arg;
ID block_arg;
NODE *kw_args;
NODE *kw_rest_arg;
NODE *opt_args;
unsigned int no_kwarg: 1;
unsigned int ruby2_keywords: 1;
unsigned int forwarding: 1;
VALUE imemo;
};
struct rb_ary_pattern_info {
NODE *pre_args;
NODE *rest_arg;
NODE *post_args;
};
struct rb_fnd_pattern_info {
NODE *pre_rest_arg;
NODE *args;
NODE *post_rest_arg;
};
struct parser_params;
void *rb_parser_malloc(struct parser_params *, size_t);
void *rb_parser_realloc(struct parser_params *, void *, size_t);
void *rb_parser_calloc(struct parser_params *, size_t, size_t);
void rb_parser_free(struct parser_params *, void *);
PRINTF_ARGS(void rb_parser_printf(struct parser_params *parser, const char *fmt, ...), 2, 3);
void rb_ast_node_type_change(NODE *n, enum node_type type);
RUBY_SYMBOL_EXPORT_END
static inline VALUE
rb_node_set_type(NODE *n, enum node_type t)
{
#if RUBY_DEBUG
rb_ast_node_type_change(n, t);
#endif
return nd_init_type(n, t);
}
2021-12-04 04:35:44 +03:00
static inline bool
nd_type_p(const NODE *n, enum node_type t)
{
return (enum node_type)nd_type(n) == t;
}
#if defined(__cplusplus)
#if 0
{ /* satisfy cc-mode */
#endif
} /* extern "C" { */
#endif
#endif /* RUBY_NODE_H */