Manage `rb_strterm_t` without imemo

gc.c

@@ -7066,7 +7066,6 @@ gc_mark_imemo(rb_objspace_t *objspace, VALUE obj)
         rb_ast_mark(&RANY(obj)->as.imemo.ast);
         return;
       case imemo_parser_strterm:
-        rb_strterm_mark(obj);
         return;
       case imemo_callinfo:
         return;

internal/imemo.h

@@ -129,7 +129,6 @@ struct MEMO {
 typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t;
 rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt);
 struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc);
-void rb_strterm_mark(VALUE obj);
 static inline enum imemo_type imemo_type(VALUE imemo);
 static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type);
 static inline bool imemo_throw_data_p(VALUE imemo);

internal/parse.h

@@ -36,7 +36,6 @@ typedef struct rb_strterm_heredoc_struct {
     uint8_t quote;
     uint8_t func;
 } rb_strterm_heredoc_t;
-STATIC_ASSERT(rb_strterm_heredoc_t, sizeof(rb_strterm_heredoc_t) <= 4 * SIZEOF_VALUE);
 
 #define HERETERM_LENGTH_MAX UINT_MAX
 

parse.y

@@ -157,23 +157,6 @@ static NODE *reg_named_capture_assign(struct parser_params* p, VALUE regexp, con
 VALUE rb_io_gets_internal(VALUE io);
 
 VALUE rb_node_case_when_optimizable_literal(const NODE *const node);
-
-static int
-strterm_is_heredoc(VALUE strterm)
-{
-    return ((rb_strterm_t *)strterm)->flags & STRTERM_HEREDOC;
-}
-
-static VALUE
-new_strterm(VALUE v1, VALUE v2, VALUE v3, VALUE v0, int heredoc)
-{
-    rb_strterm_t *imemo = (rb_strterm_t *)rb_imemo_new(imemo_parser_strterm, v1, v2, v3, v0);
-    if (heredoc) {
-        imemo->flags |= STRTERM_HEREDOC;
-    }
-
-    return (VALUE)imemo;
-}
 #endif /* !UNIVERSAL_PARSER */
 
 static inline int
@@ -7625,6 +7608,30 @@ parser_str_new(struct parser_params *p, const char *ptr, long len, rb_encoding *
     return str;
 }
 
+static int
+strterm_is_heredoc(rb_strterm_t *strterm)
+{
+    return strterm->flags & STRTERM_HEREDOC;
+}
+
+static rb_strterm_t *
+new_strterm(struct parser_params *p, int func, int term, int paren)
+{
+    rb_strterm_t *strterm = ZALLOC(rb_strterm_t);
+    strterm->u.literal.func = func;
+    strterm->u.literal.term = term;
+    strterm->u.literal.paren = paren;
+    return strterm;
+}
+
+static rb_strterm_t *
+new_heredoc(struct parser_params *p)
+{
+    rb_strterm_t *strterm = ZALLOC(rb_strterm_t);
+    strterm->flags |= STRTERM_HEREDOC;
+    return strterm;
+}
+
 #define peek(p,c) peek_n(p, (c), 0)
 #define peek_n(p,c,n) (!lex_eol_n_p(p, n) && (c) == (unsigned char)(p)->lex.pcur[n])
 #define peekc(p) peekc_n(p, 0)
@@ -7851,7 +7858,7 @@ tokadd_codepoint(struct parser_params *p, rb_encoding **encp,
     int codepoint = (int)ruby_scan_hex(p->lex.pcur, wide ? p->lex.pend - p->lex.pcur : 4, &numlen);
     p->lex.pcur += numlen;
     if (p->lex.strterm == NULL ||
-        (strterm_is_heredoc((VALUE)p->lex.strterm)) ||
+        strterm_is_heredoc(p->lex.strterm) ||
         (p->lex.strterm->u.literal.func != str_regexp)) {
         if (wide ? (numlen == 0 || numlen > 6) : (numlen < 4))  {
             literal_flush(p, p->lex.pcur);
@@ -8435,9 +8442,7 @@ tokadd_string(struct parser_params *p,
     return c;
 }
 
-/* imemo_parser_strterm for literal */
-#define NEW_STRTERM(func, term, paren) \
-    (rb_strterm_t *)new_strterm((VALUE)(func), (VALUE)(paren), (VALUE)(term), 0, 0)
+#define NEW_STRTERM(func, term, paren) new_strterm(p, func, term, paren)
 
 #ifdef RIPPER
 static void
@@ -8548,6 +8553,7 @@ parser_peek_variable_name(struct parser_params *p)
 static inline enum yytokentype
 parser_string_term(struct parser_params *p, int func)
 {
+    xfree(p->lex.strterm);
     p->lex.strterm = 0;
     if (func & STR_FUNC_REGEXP) {
         set_yylval_num(regx_options(p));
@@ -8578,6 +8584,7 @@ parse_string(struct parser_params *p, rb_strterm_literal_t *quote)
     if (func & STR_FUNC_TERM) {
         if (func & STR_FUNC_QWORDS) nextc(p); /* delayed term */
         SET_LEX_STATE(EXPR_END);
+        xfree(p->lex.strterm);
         p->lex.strterm = 0;
         return func & STR_FUNC_REGEXP ? tREGEXP_END : tSTRING_END;
     }
@@ -8624,6 +8631,7 @@ parse_string(struct parser_params *p, rb_strterm_literal_t *quote)
             if (func & STR_FUNC_QWORDS) {
                 /* no content to add, bailing out here */
                 unterminated_literal("unterminated list meets end of file");
+                xfree(p->lex.strterm);
                 p->lex.strterm = 0;
                 return tSTRING_END;
             }
@@ -8714,13 +8722,15 @@ heredoc_identifier(struct parser_params *p)
     dispatch_scan_event(p, tHEREDOC_BEG);
     lex_goto_eol(p);
 
-    p->lex.strterm = (rb_strterm_t *)new_strterm(0, 0, 0, p->lex.lastline, 1);
+    p->lex.strterm = new_heredoc(p);
     rb_strterm_heredoc_t *here = &p->lex.strterm->u.heredoc;
     here->offset = offset;
     here->sourceline = p->ruby_sourceline;
-    here->length = (int)len;
+    here->length = (unsigned)len;
     here->quote = quote;
     here->func = func;
+    here->lastline = p->lex.lastline;
+    rb_ast_add_mark_object(p->ast, p->lex.lastline);
 
     token_flush(p);
     p->heredoc_indent = indent;
@@ -8732,6 +8742,7 @@ static void
 heredoc_restore(struct parser_params *p, rb_strterm_heredoc_t *here)
 {
     VALUE line;
+    rb_strterm_t *term = p->lex.strterm;
 
     p->lex.strterm = 0;
     line = here->lastline;
@@ -8744,6 +8755,7 @@ heredoc_restore(struct parser_params *p, rb_strterm_heredoc_t *here)
     p->ruby_sourceline = (int)here->sourceline;
     if (p->eofp) p->lex.nextline = Qnil;
     p->eofp = 0;
+    xfree(term);
 }
 
 static int
@@ -9007,7 +9019,6 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here)
         compile_error(p, "can't find string \"%.*s\" anywhere before EOF",
                       (int)len, eos);
         token_flush(p);
-        p->lex.strterm = 0;
         SET_LEX_STATE(EXPR_END);
         return tSTRING_END;
     }
@@ -9027,7 +9038,6 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here)
       restore:
         heredoc_restore(p, &p->lex.strterm->u.heredoc);
         token_flush(p);
-        p->lex.strterm = 0;
         SET_LEX_STATE(EXPR_END);
         return tSTRING_END;
     }
@@ -10417,7 +10427,7 @@ parser_yylex(struct parser_params *p)
     int token_seen = p->token_seen;
 
     if (p->lex.strterm) {
-        if (strterm_is_heredoc((VALUE)p->lex.strterm)) {
+        if (strterm_is_heredoc(p->lex.strterm)) {
             token_flush(p);
             return here_document(p, &p->lex.strterm->u.heredoc);
         }
@@ -15417,7 +15427,6 @@ rb_ruby_parser_mark(void *ptr)
     rb_gc_mark(p->lex.lastline);
     rb_gc_mark(p->lex.nextline);
     rb_gc_mark(p->ruby_sourcefile_string);
-    rb_gc_mark((VALUE)p->lex.strterm);
     rb_gc_mark((VALUE)p->ast);
     rb_gc_mark(p->case_labels);
     rb_gc_mark(p->delayed.token);
@@ -15710,16 +15719,6 @@ rb_parser_set_yydebug(VALUE self, VALUE flag)
     rb_ruby_parser_set_yydebug(p, RTEST(flag));
     return flag;
 }
-
-void
-rb_strterm_mark(VALUE obj)
-{
-    rb_strterm_t *strterm = (rb_strterm_t*)obj;
-    if (RBASIC(obj)->flags & STRTERM_HEREDOC) {
-        rb_strterm_heredoc_t *heredoc = &strterm->u.heredoc;
-        rb_gc_mark(heredoc->lastline);
-    }
-}
 #endif /* !UNIVERSAL_PARSER */
 
 VALUE

ruby_parser.c

@@ -473,39 +473,12 @@ zalloc(size_t elemsiz)
     return ruby_xcalloc(1, elemsiz);
 }
 
-static VALUE
-new_strterm(VALUE v1, VALUE v2, VALUE v3, VALUE v0, int heredoc)
-{
-    rb_strterm_t *imemo = (rb_strterm_t *)rb_imemo_new(imemo_parser_strterm, v1, v2, v3, v0);
-    if (heredoc) {
-        imemo->flags |= STRTERM_HEREDOC;
-    }
-
-    return (VALUE)imemo;
-}
-
-static int
-strterm_is_heredoc(VALUE strterm)
-{
-    return ((rb_strterm_t *)strterm)->flags & STRTERM_HEREDOC;
-}
-
 static void
 gc_guard(VALUE obj)
 {
     RB_GC_GUARD(obj);
 }
 
-void
-rb_strterm_mark(VALUE obj)
-{
-    rb_strterm_t *strterm = (rb_strterm_t*)obj;
-    if (RBASIC(obj)->flags & STRTERM_HEREDOC) {
-        rb_strterm_heredoc_t *heredoc = &strterm->u.heredoc;
-        rb_gc_mark(heredoc->lastline);
-    }
-}
-
 static rb_imemo_tmpbuf_t *
 tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt)
 {
@@ -564,8 +537,6 @@ rb_parser_config_initialize(rb_parser_config_t *config)
     config->nonempty_memcpy = nonempty_memcpy;
     config->xmalloc_mul_add = rb_xmalloc_mul_add;
 
-    config->new_strterm = new_strterm;
-    config->strterm_is_heredoc = strterm_is_heredoc;
     config->tmpbuf_parser_heap = tmpbuf_parser_heap;
     config->ast_new = ast_new;
 

rubyparser.h

@@ -1143,9 +1143,6 @@ typedef struct rb_parser_config_struct {
     void *(*xmalloc_mul_add)(size_t x, size_t y, size_t z);
 
     /* imemo */
-    // TODO: Should it return `rb_strterm_t *'?
-    VALUE (*new_strterm)(VALUE v1, VALUE v2, VALUE v3, VALUE v0, int heredoc);
-    int (*strterm_is_heredoc)(VALUE strterm);
     rb_imemo_tmpbuf_t *(*tmpbuf_parser_heap)(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt);
     rb_ast_t *(*ast_new)(VALUE nb);
 

universal_parser.c

@@ -111,8 +111,6 @@ struct rb_imemo_tmpbuf_struct {
 #undef MEMCPY
 #define MEMCPY(p1,p2,type,n) (p->config->nonempty_memcpy((p1), (p2), sizeof(type), (n)))
 
-#define new_strterm p->config->new_strterm
-#define strterm_is_heredoc p->config->strterm_is_heredoc
 #define rb_imemo_tmpbuf_parser_heap p->config->tmpbuf_parser_heap
 
 #define compile_callback         p->config->compile_callback