diff --git a/array.c b/array.c index 0b2be722ea..bbe11fee7f 100644 --- a/array.c +++ b/array.c @@ -14,12 +14,14 @@ #include "ruby/encoding.h" #include "ruby/util.h" #include "ruby/st.h" -#include "internal.h" #include "probes.h" #include "id.h" #include "debug_counter.h" +#include "gc.h" +#include "transient_heap.h" +#include "internal.h" -#ifndef ARRAY_DEBUG +#if !ARRAY_DEBUG # define NDEBUG #endif #include "ruby_assert.h" @@ -42,17 +44,21 @@ VALUE rb_cArray; #define ARY_HEAP_PTR(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.ptr) #define ARY_HEAP_LEN(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.len) +#define ARY_HEAP_CAPA(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.aux.capa) + #define ARY_EMBED_PTR(a) (assert(ARY_EMBED_P(a)), RARRAY(a)->as.ary) #define ARY_EMBED_LEN(a) \ (assert(ARY_EMBED_P(a)), \ (long)((RBASIC(a)->flags >> RARRAY_EMBED_LEN_SHIFT) & \ (RARRAY_EMBED_LEN_MASK >> RARRAY_EMBED_LEN_SHIFT))) -#define ARY_HEAP_SIZE(a) (assert(!ARY_EMBED_P(a)), assert(ARY_OWNS_HEAP_P(a)), RARRAY(a)->as.heap.aux.capa * sizeof(VALUE)) +#define ARY_HEAP_SIZE(a) (assert(!ARY_EMBED_P(a)), assert(ARY_OWNS_HEAP_P(a)), ARY_HEAP_CAPA(a) * sizeof(VALUE)) #define ARY_OWNS_HEAP_P(a) (!FL_TEST((a), ELTS_SHARED|RARRAY_EMBED_FLAG)) #define FL_SET_EMBED(a) do { \ assert(!ARY_SHARED_P(a)); \ FL_SET((a), RARRAY_EMBED_FLAG); \ + FL_UNSET_RAW((a), RARRAY_TRANSIENT_FLAG); \ + ary_verify(a); \ } while (0) #define FL_UNSET_EMBED(ary) FL_UNSET((ary), RARRAY_EMBED_FLAG|RARRAY_EMBED_LEN_MASK) #define FL_SET_SHARED(ary) do { \ @@ -102,7 +108,7 @@ VALUE rb_cArray; } while (0) #define ARY_CAPA(ary) (ARY_EMBED_P(ary) ? RARRAY_EMBED_LEN_MAX : \ - ARY_SHARED_ROOT_P(ary) ? RARRAY_LEN(ary) : RARRAY(ary)->as.heap.aux.capa) + ARY_SHARED_ROOT_P(ary) ? RARRAY_LEN(ary) : ARY_HEAP_CAPA(ary)) #define ARY_SET_CAPA(ary, n) do { \ assert(!ARY_EMBED_P(ary)); \ assert(!ARY_SHARED_P(ary)); \ @@ -130,11 +136,82 @@ VALUE rb_cArray; } while (0) #define FL_SET_SHARED_ROOT(ary) do { \ assert(!ARY_EMBED_P(ary)); \ + assert(!RARRAY_TRANSIENT_P(ary)); \ FL_SET((ary), RARRAY_SHARED_ROOT_FLAG); \ } while (0) #define ARY_SET(a, i, v) RARRAY_ASET((assert(!ARY_SHARED_P(a)), (a)), (i), (v)) + +#if ARRAY_DEBUG +#define ary_verify(ary) ary_verify_(ary, __FILE__, __LINE__) + +static VALUE +ary_verify_(VALUE ary, const char *file, int line) +{ + assert(RB_TYPE_P(ary, T_ARRAY)); + + if (FL_TEST(ary, ELTS_SHARED)) { + VALUE root = RARRAY(ary)->as.heap.aux.shared; + const VALUE *ptr = ARY_HEAP_PTR(ary); + const VALUE *root_ptr = RARRAY_CONST_PTR_TRANSIENT(root); + long len = ARY_HEAP_LEN(ary), root_len = RARRAY_LEN(root); + assert(FL_TEST(root, RARRAY_SHARED_ROOT_FLAG)); + assert(root_ptr <= ptr && ptr + len <= root_ptr + root_len); + ary_verify(root); + } + else if (ARY_EMBED_P(ary)) { + assert(!RARRAY_TRANSIENT_P(ary)); + assert(!ARY_SHARED_P(ary)); + assert(RARRAY_LEN(ary) <= RARRAY_EMBED_LEN_MAX); + } + else { +#if 1 + const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary); + long i, len = RARRAY_LEN(ary); + volatile VALUE v; + if (len > 1) len = 1; /* check only HEAD */ + for (i=0; ias.heap.ptr, VALUE, new_capa, old_capa); + } + ary_verify(ary); +} + +static inline void +rb_ary_transient_heap_evacuate_(VALUE ary, int transient, int promote) +{ + if (transient) { + VALUE *new_ptr; + const VALUE *old_ptr = ARY_HEAP_PTR(ary); + long capa = ARY_HEAP_CAPA(ary); + long len = ARY_HEAP_LEN(ary); + + if (ARY_SHARED_ROOT_P(ary)) { + capa = len; + } + + assert(ARY_OWNS_HEAP_P(ary)); + assert(RARRAY_TRANSIENT_P(ary)); + assert(!ARY_PTR_USING_P(ary)); + + if (promote) { + new_ptr = ALLOC_N(VALUE, capa); + FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG); + } + else { + new_ptr = ary_heap_alloc(ary, capa); + } + + MEMCPY(new_ptr, old_ptr, VALUE, capa); + /* do not use ARY_SET_PTR() because they assert !frozen */ + RARRAY(ary)->as.heap.ptr = new_ptr; + } + + ary_verify(ary); +} + +void +rb_ary_transient_heap_evacuate(VALUE ary, int promote) +{ + rb_ary_transient_heap_evacuate_(ary, RARRAY_TRANSIENT_P(ary), promote); +} + +void +rb_ary_detransient(VALUE ary) +{ + assert(RARRAY_TRANSIENT_P(ary)); + rb_ary_transient_heap_evacuate_(ary, TRUE, TRUE); +} + static void ary_resize_capa(VALUE ary, long capacity) { assert(RARRAY_LEN(ary) <= capacity); assert(!OBJ_FROZEN(ary)); assert(!ARY_SHARED_P(ary)); + if (capacity > RARRAY_EMBED_LEN_MAX) { if (ARY_EMBED_P(ary)) { long len = ARY_EMBED_LEN(ary); - VALUE *ptr = ALLOC_N(VALUE, (capacity)); + VALUE *ptr = ary_heap_alloc(ary, capacity); + MEMCPY(ptr, ARY_EMBED_PTR(ary), VALUE, len); FL_UNSET_EMBED(ary); ARY_SET_PTR(ary, ptr); ARY_SET_HEAP_LEN(ary, len); } else { - SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, RARRAY(ary)->as.heap.aux.capa); + ary_heap_realloc(ary, capacity); } - ARY_SET_CAPA(ary, (capacity)); + ARY_SET_CAPA(ary, capacity); } else { if (!ARY_EMBED_P(ary)) { - long len = RARRAY_LEN(ary); - const VALUE *ptr = RARRAY_CONST_PTR(ary); + long len = ARY_HEAP_LEN(ary); + long old_capa = ARY_HEAP_CAPA(ary); + const VALUE *ptr = ARY_HEAP_PTR(ary); if (len > capacity) len = capacity; MEMCPY((VALUE *)RARRAY(ary)->as.ary, ptr, VALUE, len); + ary_heap_free_ptr(ary, ptr, old_capa); + FL_SET_EMBED(ary); ARY_SET_LEN(ary, len); - ruby_sized_xfree((VALUE *)ptr, RARRAY(ary)->as.heap.aux.capa); } } + + ary_verify(ary); } static inline void ary_shrink_capa(VALUE ary) { long capacity = ARY_HEAP_LEN(ary); - long old_capa = RARRAY(ary)->as.heap.aux.capa; + long old_capa = ARY_HEAP_CAPA(ary); assert(!ARY_SHARED_P(ary)); assert(old_capa >= capacity); - if (old_capa > capacity) - SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, old_capa); + if (old_capa > capacity) ary_heap_realloc(ary, capacity); + + ary_verify(ary); } static void @@ -253,6 +448,8 @@ ary_double_capa(VALUE ary, long min) } new_capa += min; ary_resize_capa(ary, new_capa); + + ary_verify(ary); } static void @@ -308,6 +505,7 @@ static inline void rb_ary_modify_check(VALUE ary) { rb_check_frozen(ary); + ary_verify(ary); } void @@ -317,6 +515,9 @@ rb_ary_modify(VALUE ary) if (ARY_SHARED_P(ary)) { long shared_len, len = RARRAY_LEN(ary); VALUE shared = ARY_SHARED(ary); + + ary_verify(shared); + if (len <= RARRAY_EMBED_LEN_MAX) { const VALUE *ptr = ARY_HEAP_PTR(ary); FL_UNSET_SHARED(ary); @@ -326,9 +527,9 @@ rb_ary_modify(VALUE ary) ARY_SET_EMBED_LEN(ary, len); } else if (ARY_SHARED_OCCUPIED(shared) && len > ((shared_len = RARRAY_LEN(shared))>>1)) { - long shift = RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared); + long shift = RARRAY_CONST_PTR_TRANSIENT(ary) - RARRAY_CONST_PTR_TRANSIENT(shared); FL_UNSET_SHARED(ary); - ARY_SET_PTR(ary, RARRAY_CONST_PTR(shared)); + ARY_SET_PTR(ary, RARRAY_CONST_PTR_TRANSIENT(shared)); ARY_SET_CAPA(ary, shared_len); RARRAY_PTR_USE(ary, ptr, { MEMMOVE(ptr, ptr+shift, VALUE, len); @@ -337,8 +538,8 @@ rb_ary_modify(VALUE ary) rb_ary_decrement_share(shared); } else { - VALUE *ptr = ALLOC_N(VALUE, len); - MEMCPY(ptr, RARRAY_CONST_PTR(ary), VALUE, len); + VALUE *ptr = ary_heap_alloc(ary, len); + MEMCPY(ptr, ARY_HEAP_PTR(ary), VALUE, len); rb_ary_unshare(ary); ARY_SET_CAPA(ary, len); ARY_SET_PTR(ary, ptr); @@ -346,6 +547,7 @@ rb_ary_modify(VALUE ary) rb_gc_writebarrier_remember(ary); } + ary_verify(ary); } static VALUE @@ -362,8 +564,11 @@ ary_ensure_room_for_push(VALUE ary, long add_len) if (new_len > RARRAY_EMBED_LEN_MAX) { VALUE shared = ARY_SHARED(ary); if (ARY_SHARED_OCCUPIED(shared)) { - if (RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared) + new_len <= RARRAY_LEN(shared)) { + if (ARY_HEAP_PTR(ary) - RARRAY_CONST_PTR_TRANSIENT(shared) + new_len <= RARRAY_LEN(shared)) { rb_ary_modify_check(ary); + + ary_verify(ary); + ary_verify(shared); return shared; } else { @@ -373,10 +578,12 @@ ary_ensure_room_for_push(VALUE ary, long add_len) if (new_len > capa - (capa >> 6)) { ary_double_capa(ary, new_len); } + ary_verify(ary); return ary; } } } + ary_verify(ary); rb_ary_modify(ary); } else { @@ -387,6 +594,7 @@ ary_ensure_room_for_push(VALUE ary, long add_len) ary_double_capa(ary, new_len); } + ary_verify(ary); return ary; } @@ -459,7 +667,7 @@ ary_new(VALUE klass, long capa) ary = ary_alloc(klass); if (capa > RARRAY_EMBED_LEN_MAX) { - ptr = ALLOC_N(VALUE, capa); + ptr = ary_heap_alloc(ary, capa); FL_UNSET_EMBED(ary); ARY_SET_PTR(ary, ptr); ARY_SET_CAPA(ary, capa); @@ -523,7 +731,9 @@ rb_ary_new_from_values(long n, const VALUE *elts) VALUE rb_ary_tmp_new(long capa) { - return ary_new(0, capa); + VALUE ary = ary_new(0, capa); + rb_ary_transient_heap_evacuate(ary, TRUE); + return ary; } VALUE @@ -532,6 +742,7 @@ rb_ary_tmp_new_fill(long capa) VALUE ary = ary_new(0, capa); ary_memfill(ary, 0, capa, Qnil); ARY_SET_LEN(ary, capa); + rb_ary_transient_heap_evacuate(ary, TRUE); return ary; } @@ -539,8 +750,13 @@ void rb_ary_free(VALUE ary) { if (ARY_OWNS_HEAP_P(ary)) { - RB_DEBUG_COUNTER_INC(obj_ary_ptr); - ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary)); + if (RARRAY_TRANSIENT_P(ary)) { + RB_DEBUG_COUNTER_INC(obj_ary_transient); + } + else { + RB_DEBUG_COUNTER_INC(obj_ary_ptr); + ary_heap_free(ary); + } } else { RB_DEBUG_COUNTER_INC(obj_ary_embed); @@ -563,13 +779,15 @@ ary_discard(VALUE ary) { rb_ary_free(ary); RBASIC(ary)->flags |= RARRAY_EMBED_FLAG; - RBASIC(ary)->flags &= ~RARRAY_EMBED_LEN_MASK; + RBASIC(ary)->flags &= ~(RARRAY_EMBED_LEN_MASK | RARRAY_TRANSIENT_FLAG); } static VALUE ary_make_shared(VALUE ary) { assert(!ARY_EMBED_P(ary)); + ary_verify(ary); + if (ARY_SHARED_P(ary)) { return ARY_SHARED(ary); } @@ -577,6 +795,7 @@ ary_make_shared(VALUE ary) return ary; } else if (OBJ_FROZEN(ary)) { + rb_ary_transient_heap_evacuate(ary, TRUE); ary_shrink_capa(ary); FL_SET_SHARED_ROOT(ary); ARY_SET_SHARED_NUM(ary, 1); @@ -584,17 +803,24 @@ ary_make_shared(VALUE ary) } else { long capa = ARY_CAPA(ary), len = RARRAY_LEN(ary); + const VALUE *ptr; NEWOBJ_OF(shared, struct RArray, 0, T_ARRAY | (RGENGC_WB_PROTECTED_ARRAY ? FL_WB_PROTECTED : 0)); - FL_UNSET_EMBED(shared); + rb_ary_transient_heap_evacuate(ary, TRUE); + ptr = ARY_HEAP_PTR(ary); + + FL_UNSET_EMBED(shared); ARY_SET_LEN((VALUE)shared, capa); - ARY_SET_PTR((VALUE)shared, RARRAY_CONST_PTR(ary)); + ARY_SET_PTR((VALUE)shared, ptr); ary_mem_clear((VALUE)shared, len, capa - len); FL_SET_SHARED_ROOT(shared); ARY_SET_SHARED_NUM((VALUE)shared, 1); FL_SET_SHARED(ary); ARY_SET_SHARED(ary, (VALUE)shared); OBJ_FREEZE(shared); + + ary_verify((VALUE)shared); + ary_verify(ary); return (VALUE)shared; } } @@ -606,7 +832,7 @@ ary_make_substitution(VALUE ary) if (len <= RARRAY_EMBED_LEN_MAX) { VALUE subst = rb_ary_new2(len); - ary_memcpy(subst, 0, len, RARRAY_CONST_PTR(ary)); + ary_memcpy(subst, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary)); ARY_SET_EMBED_LEN(subst, len); return subst; } @@ -729,8 +955,8 @@ rb_ary_initialize(int argc, VALUE *argv, VALUE ary) rb_ary_modify(ary); if (argc == 0) { - if (ARY_OWNS_HEAP_P(ary) && RARRAY_CONST_PTR(ary) != 0) { - ruby_sized_xfree((void *)RARRAY_CONST_PTR(ary), ARY_HEAP_SIZE(ary)); + if (ARY_OWNS_HEAP_P(ary) && ARY_HEAP_PTR(ary) != NULL) { + ary_heap_free(ary); } rb_ary_unshare_safe(ary); FL_SET_EMBED(ary); @@ -837,7 +1063,7 @@ ary_make_partial(VALUE ary, VALUE klass, long offset, long len) if (len <= RARRAY_EMBED_LEN_MAX) { VALUE result = ary_alloc(klass); - ary_memcpy(result, 0, len, RARRAY_CONST_PTR(ary) + offset); + ary_memcpy(result, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary) + offset); ARY_SET_EMBED_LEN(result, len); return result; } @@ -846,12 +1072,15 @@ ary_make_partial(VALUE ary, VALUE klass, long offset, long len) FL_UNSET_EMBED(result); shared = ary_make_shared(ary); - ARY_SET_PTR(result, RARRAY_CONST_PTR(ary)); + ARY_SET_PTR(result, RARRAY_CONST_PTR_TRANSIENT(ary)); ARY_SET_LEN(result, RARRAY_LEN(ary)); rb_ary_set_shared(result, shared); ARY_INCREASE_PTR(result, offset); ARY_SET_LEN(result, len); + + ary_verify(shared); + ary_verify(result); return result; } } @@ -910,12 +1139,13 @@ ary_take_first_or_last(int argc, const VALUE *argv, VALUE ary, enum ary_take_pos VALUE rb_ary_push(VALUE ary, VALUE item) { - long idx = RARRAY_LEN(ary); + long idx = RARRAY_LEN((ary_verify(ary), ary)); VALUE target_ary = ary_ensure_room_for_push(ary, 1); RARRAY_PTR_USE(ary, ptr, { RB_OBJ_WRITE(target_ary, &ptr[idx], item); }); ARY_SET_LEN(ary, idx + 1); + ary_verify(ary); return ary; } @@ -967,6 +1197,7 @@ rb_ary_pop(VALUE ary) } --n; ARY_SET_LEN(ary, n); + ary_verify(ary); return RARRAY_AREF(ary, n); } @@ -1000,6 +1231,7 @@ rb_ary_pop_m(int argc, VALUE *argv, VALUE ary) rb_ary_modify_check(ary); result = ary_take_first_or_last(argc, argv, ary, ARY_TAKE_LAST); ARY_INCREASE_LEN(ary, -RARRAY_LEN(result)); + ary_verify(ary); return result; } @@ -1018,6 +1250,7 @@ rb_ary_shift(VALUE ary) MEMMOVE(ptr, ptr+1, VALUE, len-1); }); /* WB: no new reference */ ARY_INCREASE_LEN(ary, -1); + ary_verify(ary); return top; } assert(!ARY_EMBED_P(ary)); /* ARY_EMBED_LEN_MAX < ARY_DEFAULT_SIZE */ @@ -1031,6 +1264,8 @@ rb_ary_shift(VALUE ary) ARY_INCREASE_PTR(ary, 1); /* shift ptr */ ARY_INCREASE_LEN(ary, -1); + ary_verify(ary); + return top; } @@ -1101,6 +1336,7 @@ rb_ary_behead(VALUE ary, long n) } ARY_INCREASE_LEN(ary, -n); + ary_verify(ary); return ary; } @@ -1120,8 +1356,8 @@ ary_ensure_room_for_unshift(VALUE ary, int argc) VALUE shared = ARY_SHARED(ary); capa = RARRAY_LEN(shared); if (ARY_SHARED_OCCUPIED(shared) && capa > new_len) { - head = RARRAY_CONST_PTR(ary); - sharedp = RARRAY_CONST_PTR(shared); + head = RARRAY_CONST_PTR_TRANSIENT(ary); + sharedp = RARRAY_CONST_PTR_TRANSIENT(shared); goto makeroom_if_need; } } @@ -1134,11 +1370,13 @@ ary_ensure_room_for_unshift(VALUE ary, int argc) /* use shared array for big "queues" */ if (new_len > ARY_DEFAULT_SIZE * 4) { + ary_verify(ary); + /* make a room for unshifted items */ capa = ARY_CAPA(ary); ary_make_shared(ary); - head = sharedp = RARRAY_CONST_PTR(ary); + head = sharedp = RARRAY_CONST_PTR_TRANSIENT(ary); goto makeroom; makeroom_if_need: if (head - sharedp < argc) { @@ -1151,6 +1389,8 @@ ary_ensure_room_for_unshift(VALUE ary, int argc) } ARY_SET_PTR(ary, head - argc); assert(ARY_SHARED_OCCUPIED(ARY_SHARED(ary))); + + ary_verify(ary); return ARY_SHARED(ary); } else { @@ -1159,6 +1399,7 @@ ary_ensure_room_for_unshift(VALUE ary, int argc) MEMMOVE(ptr + argc, ptr, VALUE, len); }); + ary_verify(ary); return ary; } } @@ -1576,7 +1817,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen) } { - const VALUE *optr = RARRAY_CONST_PTR(ary); + const VALUE *optr = RARRAY_CONST_PTR_TRANSIENT(ary); rofs = (rptr >= optr && rptr < optr + olen) ? rptr - optr : -1; } @@ -1589,7 +1830,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen) len = beg + rlen; ary_mem_clear(ary, olen, beg - olen); if (rlen > 0) { - if (rofs != -1) rptr = RARRAY_CONST_PTR(ary) + rofs; + if (rofs != -1) rptr = RARRAY_CONST_PTR_TRANSIENT(ary) + rofs; ary_memcpy0(ary, beg, rlen, rptr, target_ary); } ARY_SET_LEN(ary, len); @@ -1613,7 +1854,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen) ARY_SET_LEN(ary, alen); } if (rlen > 0) { - if (rofs != -1) rptr = RARRAY_CONST_PTR(ary) + rofs; + if (rofs != -1) rptr = RARRAY_CONST_PTR_TRANSIENT(ary) + rofs; /* give up wb-protected ary */ MEMMOVE(RARRAY_PTR(ary) + beg, rptr, VALUE, rlen); } @@ -1673,11 +1914,12 @@ rb_ary_resize(VALUE ary, long len) } else { if (olen > len + ARY_DEFAULT_SIZE) { - SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, len, RARRAY(ary)->as.heap.aux.capa); + ary_heap_realloc(ary, len); ARY_SET_CAPA(ary, len); } ARY_SET_HEAP_LEN(ary, len); } + ary_verify(ary); return ary; } @@ -1738,7 +1980,7 @@ rb_ary_aset(int argc, VALUE *argv, VALUE ary) /* check if idx is Range */ range: rpl = rb_ary_to_ary(argv[argc-1]); - rb_ary_splice(ary, beg, len, RARRAY_CONST_PTR(rpl), RARRAY_LEN(rpl)); + rb_ary_splice(ary, beg, len, RARRAY_CONST_PTR_TRANSIENT(rpl), RARRAY_LEN(rpl)); RB_GC_GUARD(rpl); return argv[argc-1]; } @@ -1820,7 +2062,7 @@ VALUE rb_ary_each(VALUE ary) { long i; - + ary_verify(ary); RETURN_SIZED_ENUMERATOR(ary, 0, 0, ary_enum_length); for (i=0; i 0) { - const VALUE *p1 = RARRAY_CONST_PTR(ary); - VALUE *p2 = (VALUE *)RARRAY_CONST_PTR(dup) + len - 1; + const VALUE *p1 = RARRAY_CONST_PTR_TRANSIENT(ary); + VALUE *p2 = (VALUE *)RARRAY_CONST_PTR_TRANSIENT(dup) + len - 1; do *p2-- = *p1++; while (--len > 0); } ARY_SET_LEN(dup, RARRAY_LEN(ary)); @@ -2382,7 +2627,7 @@ rb_ary_rotate_m(int argc, VALUE *argv, VALUE ary) rotated = rb_ary_new2(len); if (len > 0) { cnt = rotate_count(cnt, len); - ptr = RARRAY_CONST_PTR(ary); + ptr = RARRAY_CONST_PTR_TRANSIENT(ary); len -= cnt; ary_memcpy(rotated, 0, len, ptr + cnt); ary_memcpy(rotated, len, cnt, ptr); @@ -2482,7 +2727,6 @@ rb_ary_sort_bang(VALUE ary) VALUE tmp = ary_make_substitution(ary); /* only ary refers tmp */ struct ary_sort_data data; long len = RARRAY_LEN(ary); - RBASIC_CLEAR_CLASS(tmp); data.ary = tmp; data.cmp_opt.opt_methods = 0; @@ -2515,11 +2759,11 @@ rb_ary_sort_bang(VALUE ary) rb_ary_unshare(ary); } else { - ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary)); + ary_heap_free(ary); } - ARY_SET_PTR(ary, RARRAY_CONST_PTR(tmp)); + ARY_SET_PTR(ary, ARY_HEAP_PTR(tmp)); ARY_SET_HEAP_LEN(ary, len); - ARY_SET_CAPA(ary, RARRAY_LEN(tmp)); + ARY_SET_CAPA(ary, ARY_HEAP_LEN(tmp)); } /* tmp was lost ownership for the ptr */ FL_UNSET(tmp, FL_FREEZE); @@ -2530,6 +2774,7 @@ rb_ary_sort_bang(VALUE ary) /* tmp will be GC'ed. */ RBASIC_SET_CLASS_RAW(tmp, rb_cArray); /* rb_cArray must be marked */ } + ary_verify(ary); return ary; } @@ -2833,7 +3078,7 @@ append_values_at_single(VALUE result, VALUE ary, long olen, VALUE idx) /* check if idx is Range */ else if (rb_range_beg_len(idx, &beg, &len, olen, 1)) { if (len > 0) { - const VALUE *const src = RARRAY_CONST_PTR(ary); + const VALUE *const src = RARRAY_CONST_PTR_TRANSIENT(ary); const long end = beg + len; const long prevlen = RARRAY_LEN(result); if (beg < olen) { @@ -3075,6 +3320,7 @@ rb_ary_delete(VALUE ary, VALUE item) ary_resize_smaller(ary, i2); + ary_verify(ary); return v; } @@ -3119,7 +3365,7 @@ rb_ary_delete_at(VALUE ary, long pos) MEMMOVE(ptr+pos, ptr+pos+1, VALUE, len-pos-1); }); ARY_INCREASE_LEN(ary, -1); - + ary_verify(ary); return del; } @@ -3187,7 +3433,7 @@ rb_ary_slice_bang(int argc, VALUE *argv, VALUE ary) len = orig_len - pos; } if (len == 0) return rb_ary_new2(0); - arg2 = rb_ary_new4(len, RARRAY_CONST_PTR(ary)+pos); + arg2 = rb_ary_new4(len, RARRAY_CONST_PTR_TRANSIENT(ary)+pos); RBASIC_SET_CLASS(arg2, rb_obj_class(ary)); rb_ary_splice(ary, pos, len, 0, 0); return arg2; @@ -3223,6 +3469,7 @@ ary_reject(VALUE orig, VALUE result) for (i = 0; i < RARRAY_LEN(orig); i++) { VALUE v = RARRAY_AREF(orig, i); + if (!RTEST(rb_yield(v))) { rb_ary_push(result, v); } @@ -3252,7 +3499,6 @@ static VALUE ary_reject_bang(VALUE ary) { struct select_bang_arg args; - rb_ary_modify_check(ary); args.ary = ary; args.len[0] = args.len[1] = 0; @@ -3326,6 +3572,7 @@ rb_ary_reject(VALUE ary) static VALUE rb_ary_delete_if(VALUE ary) { + ary_verify(ary); RETURN_SIZED_ENUMERATOR(ary, 0, 0, ary_enum_length); ary_reject_bang(ary); return ary; @@ -3504,14 +3751,14 @@ rb_ary_replace(VALUE copy, VALUE orig) VALUE shared = 0; if (ARY_OWNS_HEAP_P(copy)) { - RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy))); + ary_heap_free(copy); } else if (ARY_SHARED_P(copy)) { shared = ARY_SHARED(copy); FL_UNSET_SHARED(copy); } FL_SET_EMBED(copy); - ary_memcpy(copy, 0, RARRAY_LEN(orig), RARRAY_CONST_PTR(orig)); + ary_memcpy(copy, 0, RARRAY_LEN(orig), RARRAY_CONST_PTR_TRANSIENT(orig)); if (shared) { rb_ary_decrement_share(shared); } @@ -3520,16 +3767,17 @@ rb_ary_replace(VALUE copy, VALUE orig) else { VALUE shared = ary_make_shared(orig); if (ARY_OWNS_HEAP_P(copy)) { - RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy))); + ary_heap_free(copy); } else { rb_ary_unshare_safe(copy); } FL_UNSET_EMBED(copy); - ARY_SET_PTR(copy, RARRAY_CONST_PTR(orig)); - ARY_SET_LEN(copy, RARRAY_LEN(orig)); + ARY_SET_PTR(copy, ARY_HEAP_PTR(orig)); + ARY_SET_LEN(copy, ARY_HEAP_LEN(orig)); rb_ary_set_shared(copy, shared); } + ary_verify(copy); return copy; } @@ -3547,16 +3795,20 @@ VALUE rb_ary_clear(VALUE ary) { rb_ary_modify_check(ary); - ARY_SET_LEN(ary, 0); if (ARY_SHARED_P(ary)) { if (!ARY_EMBED_P(ary)) { rb_ary_unshare(ary); FL_SET_EMBED(ary); + ARY_SET_EMBED_LEN(ary, 0); } } - else if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) { - ary_resize_capa(ary, ARY_DEFAULT_SIZE * 2); + else { + ARY_SET_LEN(ary, 0); + if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) { + ary_resize_capa(ary, ARY_DEFAULT_SIZE * 2); + } } + ary_verify(ary); return ary; } @@ -3689,8 +3941,8 @@ rb_ary_plus(VALUE x, VALUE y) len = xlen + ylen; z = rb_ary_new2(len); - ary_memcpy(z, 0, xlen, RARRAY_CONST_PTR(x)); - ary_memcpy(z, xlen, ylen, RARRAY_CONST_PTR(y)); + ary_memcpy(z, 0, xlen, RARRAY_CONST_PTR_TRANSIENT(x)); + ary_memcpy(z, xlen, ylen, RARRAY_CONST_PTR_TRANSIENT(y)); ARY_SET_LEN(z, len); return z; } @@ -3700,7 +3952,7 @@ ary_append(VALUE x, VALUE y) { long n = RARRAY_LEN(y); if (n > 0) { - rb_ary_splice(x, RARRAY_LEN(x), 0, RARRAY_CONST_PTR(y), n); + rb_ary_splice(x, RARRAY_LEN(x), 0, RARRAY_CONST_PTR_TRANSIENT(y), n); } return x; } @@ -3742,6 +3994,7 @@ rb_ary_concat_multi(int argc, VALUE *argv, VALUE ary) ary_append(ary, args); } + ary_verify(ary); return ary; } @@ -3796,16 +4049,16 @@ rb_ary_times(VALUE ary, VALUE times) ary2 = ary_new(rb_obj_class(ary), len); ARY_SET_LEN(ary2, len); - ptr = RARRAY_CONST_PTR(ary); + ptr = RARRAY_CONST_PTR_TRANSIENT(ary); t = RARRAY_LEN(ary); if (0 < t) { ary_memcpy(ary2, 0, t, ptr); while (t <= len/2) { - ary_memcpy(ary2, t, t, RARRAY_CONST_PTR(ary2)); + ary_memcpy(ary2, t, t, RARRAY_CONST_PTR_TRANSIENT(ary2)); t *= 2; } if (t < len) { - ary_memcpy(ary2, t, len-t, RARRAY_CONST_PTR(ary2)); + ary_memcpy(ary2, t, len-t, RARRAY_CONST_PTR_TRANSIENT(ary2)); } } out: @@ -3891,6 +4144,7 @@ recursive_equal(VALUE ary1, VALUE ary2, int recur) if (recur) return Qtrue; /* Subtle! */ + /* rb_equal() can evacuate ptrs */ p1 = RARRAY_CONST_PTR(ary1); p2 = RARRAY_CONST_PTR(ary2); len1 = RARRAY_LEN(ary1); @@ -3941,7 +4195,7 @@ rb_ary_equal(VALUE ary1, VALUE ary2) return rb_equal(ary2, ary1); } if (RARRAY_LEN(ary1) != RARRAY_LEN(ary2)) return Qfalse; - if (RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2)) return Qtrue; + if (RARRAY_CONST_PTR_TRANSIENT(ary1) == RARRAY_CONST_PTR_TRANSIENT(ary2)) return Qtrue; return rb_exec_recursive_paired(recursive_equal, ary1, ary2, ary2); } @@ -3972,7 +4226,7 @@ rb_ary_eql(VALUE ary1, VALUE ary2) if (ary1 == ary2) return Qtrue; if (!RB_TYPE_P(ary2, T_ARRAY)) return Qfalse; if (RARRAY_LEN(ary1) != RARRAY_LEN(ary2)) return Qfalse; - if (RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2)) return Qtrue; + if (RARRAY_CONST_PTR_TRANSIENT(ary1) == RARRAY_CONST_PTR_TRANSIENT(ary2)) return Qtrue; return rb_exec_recursive_paired(recursive_eql, ary1, ary2, ary2); } @@ -4679,14 +4933,14 @@ rb_ary_compact_bang(VALUE ary) long n; rb_ary_modify(ary); - p = t = (VALUE *)RARRAY_CONST_PTR(ary); /* WB: no new reference */ + p = t = (VALUE *)RARRAY_CONST_PTR_TRANSIENT(ary); /* WB: no new reference */ end = p + RARRAY_LEN(ary); while (t < end) { if (NIL_P(*t)) t++; else *p++ = *t++; } - n = p - RARRAY_CONST_PTR(ary); + n = p - RARRAY_CONST_PTR_TRANSIENT(ary); if (RARRAY_LEN(ary) == n) { return Qnil; } @@ -4949,7 +5203,7 @@ rb_ary_shuffle_bang(int argc, VALUE *argv, VALUE ary) while (i) { long j = RAND_UPTO(i); VALUE tmp; - if (len != RARRAY_LEN(ary) || ptr != RARRAY_CONST_PTR(ary)) { + if (len != RARRAY_LEN(ary) || ptr != RARRAY_CONST_PTR_TRANSIENT(ary)) { rb_raise(rb_eRuntimeError, "modified during shuffle"); } tmp = ptr[--i]; diff --git a/common.mk b/common.mk index 42584f65a2..b5f391d150 100644 --- a/common.mk +++ b/common.mk @@ -130,6 +130,7 @@ COMMONOBJS = array.$(OBJEXT) \ thread.$(OBJEXT) \ time.$(OBJEXT) \ transcode.$(OBJEXT) \ + transient_heap.$(OBJEXT) \ util.$(OBJEXT) \ variable.$(OBJEXT) \ version.$(OBJEXT) \ @@ -2898,6 +2899,15 @@ transcode.$(OBJEXT): {$(VPATH)}st.h transcode.$(OBJEXT): {$(VPATH)}subst.h transcode.$(OBJEXT): {$(VPATH)}transcode.c transcode.$(OBJEXT): {$(VPATH)}transcode_data.h +transient_heap.$(OBJEXT): $(hdrdir)/ruby/ruby.h +transient_heap.$(OBJEXT): $(top_srcdir)/include/ruby.h +transient_heap.$(OBJEXT): {$(VPATH)}debug_counter.h +transient_heap.$(OBJEXT): {$(VPATH)}gc.h +transient_heap.$(OBJEXT): {$(VPATH)}internal.h +transient_heap.$(OBJEXT): {$(VPATH)}ruby_assert.h +transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.c +transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.h +transient_heap.$(OBJEXT): {$(VPATH)}vm_debug.h util.$(OBJEXT): $(hdrdir)/ruby/ruby.h util.$(OBJEXT): $(top_srcdir)/include/ruby.h util.$(OBJEXT): {$(VPATH)}config.h diff --git a/compile.c b/compile.c index 4f98ac7997..58575ec929 100644 --- a/compile.c +++ b/compile.c @@ -1649,7 +1649,7 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons opt_table = ALLOC_N(VALUE, i+1); - MEMCPY(opt_table, RARRAY_CONST_PTR(labels), VALUE, i+1); + MEMCPY(opt_table, RARRAY_CONST_PTR_TRANSIENT(labels), VALUE, i+1); for (j = 0; j < i+1; j++) { opt_table[j] &= ~1; } @@ -2297,14 +2297,14 @@ iseq_set_exception_table(rb_iseq_t *iseq) struct iseq_catch_table_entry *entry; tlen = (int)RARRAY_LEN(ISEQ_COMPILE_DATA(iseq)->catch_table_ary); - tptr = RARRAY_CONST_PTR(ISEQ_COMPILE_DATA(iseq)->catch_table_ary); + tptr = RARRAY_CONST_PTR_TRANSIENT(ISEQ_COMPILE_DATA(iseq)->catch_table_ary); if (tlen > 0) { struct iseq_catch_table *table = xmalloc(iseq_catch_table_bytes(tlen)); table->size = tlen; for (i = 0; i < table->size; i++) { - ptr = RARRAY_CONST_PTR(tptr[i]); + ptr = RARRAY_CONST_PTR_TRANSIENT(tptr[i]); entry = &table->entries[i]; entry->type = (enum catch_type)(ptr[0] & 0xffff); entry->start = label_get_position((LABEL *)(ptr[1] & ~1)); diff --git a/debug_counter.h b/debug_counter.h index 3fcb562f65..c151e535b0 100644 --- a/debug_counter.h +++ b/debug_counter.h @@ -219,6 +219,11 @@ RB_DEBUG_COUNTER(heap_xmalloc) RB_DEBUG_COUNTER(heap_xrealloc) RB_DEBUG_COUNTER(heap_xfree) +/* transient_heap */ +RB_DEBUG_COUNTER(theap_alloc) +RB_DEBUG_COUNTER(theap_alloc_fail) +RB_DEBUG_COUNTER(theap_evacuate) + /* load (not implemented yet) */ /* RB_DEBUG_COUNTER(load_files) diff --git a/enum.c b/enum.c index 71e7a74d35..3be6941c3a 100644 --- a/enum.c +++ b/enum.c @@ -14,6 +14,7 @@ #include "ruby/util.h" #include "id.h" #include "symbol.h" +#include "transient_heap.h" #include @@ -1171,9 +1172,10 @@ enum_sort_by(VALUE obj) rb_ary_concat(ary, buf); } if (RARRAY_LEN(ary) > 2) { + rb_ary_transient_heap_evacuate(ary, TRUE); /* should be malloc heap */ RARRAY_PTR_USE(ary, ptr, - ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE), - sort_by_cmp, (void *)ary)); + ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE), + sort_by_cmp, (void *)ary)); } if (RBASIC(ary)->klass) { rb_raise(rb_eRuntimeError, "sort_by reentered"); diff --git a/gc.c b/gc.c index 00f51c9657..57619af9f3 100644 --- a/gc.c +++ b/gc.c @@ -35,6 +35,7 @@ #include #include "ruby_assert.h" #include "debug_counter.h" +#include "transient_heap.h" #include "mjit.h" #undef rb_data_object_wrap @@ -845,8 +846,6 @@ static void rb_objspace_call_finalizer(rb_objspace_t *objspace); static VALUE define_final0(VALUE obj, VALUE block); static void negative_size_allocation_error(const char *); -static void *aligned_malloc(size_t, size_t); -static void aligned_free(void *); static void init_mark_stack(mark_stack_t *stack); @@ -1190,6 +1189,7 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *pag { MARK_IN_BITMAP(&page->uncollectible_bits[0], obj); objspace->rgengc.old_objects++; + rb_transient_heap_promote(obj); #if RGENGC_PROFILE >= 2 objspace->profile.total_promoted_count++; @@ -1486,7 +1486,7 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page) { heap_allocated_pages--; objspace->profile.total_freed_pages++; - aligned_free(GET_PAGE_BODY(page->start)); + rb_aligned_free(GET_PAGE_BODY(page->start)); free(page); } @@ -1524,7 +1524,7 @@ heap_page_allocate(rb_objspace_t *objspace) int limit = HEAP_PAGE_OBJ_LIMIT; /* assign heap_page body (contains heap_page_header and RVALUEs) */ - page_body = (struct heap_page_body *)aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE); + page_body = (struct heap_page_body *)rb_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE); if (page_body == 0) { rb_memerror(); } @@ -1532,7 +1532,7 @@ heap_page_allocate(rb_objspace_t *objspace) /* assign heap_page entry */ page = (struct heap_page *)calloc(1, sizeof(struct heap_page)); if (page == 0) { - aligned_free(page_body); + rb_aligned_free(page_body); rb_memerror(); } @@ -4515,6 +4515,7 @@ static void gc_mark_ptr(rb_objspace_t *objspace, VALUE obj) { if (LIKELY(objspace->mark_func_data == NULL)) { + if (RB_TYPE_P(obj, T_NONE)) rb_bug("..."); rgengc_check_relation(objspace, obj); if (!gc_mark_set(objspace, obj)) return; /* already marked */ gc_aging(objspace, obj); @@ -4672,14 +4673,22 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) case T_ARRAY: if (FL_TEST(obj, ELTS_SHARED)) { - gc_mark(objspace, any->as.array.as.heap.aux.shared); + VALUE root = any->as.array.as.heap.aux.shared; + gc_mark(objspace, root); } else { long i, len = RARRAY_LEN(obj); - const VALUE *ptr = RARRAY_CONST_PTR(obj); + const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(obj); for (i=0; i < len; i++) { - gc_mark(objspace, *ptr++); + gc_mark(objspace, ptr[i]); } + + if (objspace->mark_func_data == NULL) { + if (!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG) && + RARRAY_TRANSIENT_P(obj)) { + rb_transient_heap_mark(obj, ptr); + } + } } break; @@ -5455,6 +5464,13 @@ rb_gc_verify_internal_consistency(void) gc_verify_internal_consistency(Qnil); } +static VALUE +gc_verify_transient_heap_internal_consistency(VALUE dmy) +{ + rb_transient_heap_verify(); + return Qnil; +} + /* marks */ static void @@ -5671,6 +5687,8 @@ gc_marks_finish(rb_objspace_t *objspace) #endif } + rb_transient_heap_finish_marking(); + gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0); return TRUE; @@ -6562,6 +6580,7 @@ gc_start(rb_objspace_t *objspace, int reason) objspace->profile.heap_used_at_gc_start = heap_allocated_pages; gc_prof_setup_new_record(objspace, reason); gc_reset_malloc_info(objspace); + rb_transient_heap_start_marking(do_full_mark); gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */); GC_ASSERT(during_gc); @@ -7812,8 +7831,8 @@ rb_memerror(void) EC_JUMP_TAG(ec, TAG_RAISE); } -static void * -aligned_malloc(size_t alignment, size_t size) +void * +rb_aligned_malloc(size_t alignment, size_t size) { void *res; @@ -7846,8 +7865,8 @@ aligned_malloc(size_t alignment, size_t size) return res; } -static void -aligned_free(void *ptr) +void +rb_aligned_free(void *ptr) { #if defined __MINGW32__ __mingw_aligned_free(ptr); @@ -9551,13 +9570,21 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj) #if USE_RGENGC const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags); - snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s", - (void *)obj, age, - C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"), - C(RVALUE_MARK_BITMAP(obj), "M"), - C(RVALUE_MARKING_BITMAP(obj), "R"), - C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"), - obj_type_name(obj)); + if (is_pointer_to_heap(&rb_objspace, (void *)obj)) { + snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s", + (void *)obj, age, + C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"), + C(RVALUE_MARK_BITMAP(obj), "M"), + C(RVALUE_MARKING_BITMAP(obj), "R"), + C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"), + obj_type_name(obj)); + } + else { + /* fake */ + snprintf(buff, buff_size, "%p [%dXXXX] %s", + (void *)obj, age, + obj_type_name(obj)); + } #else snprintf(buff, buff_size, "%p [%s] %s", (void *)obj, @@ -9587,10 +9614,25 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj) UNEXPECTED_NODE(rb_raw_obj_info); break; case T_ARRAY: - snprintf(buff, buff_size, "%s [%s%s] len: %d", buff, - C(ARY_EMBED_P(obj), "E"), - C(ARY_SHARED_P(obj), "S"), - (int)RARRAY_LEN(obj)); + if (FL_TEST(obj, ELTS_SHARED)) { + snprintf(buff, buff_size, "%s shared -> %s", buff, + rb_obj_info(RARRAY(obj)->as.heap.aux.shared)); + } + else if (FL_TEST(obj, RARRAY_EMBED_FLAG)) { + snprintf(buff, buff_size, "%s [%s%s] len: %d (embed)", buff, + C(ARY_EMBED_P(obj), "E"), + C(ARY_SHARED_P(obj), "S"), + (int)RARRAY_LEN(obj)); + } + else { + snprintf(buff, buff_size, "%s [%s%s%s] len: %d, capa:%d ptr:%p", buff, + C(ARY_EMBED_P(obj), "E"), + C(ARY_SHARED_P(obj), "S"), + C(RARRAY_TRANSIENT_P(obj), "T"), + (int)RARRAY_LEN(obj), + ARY_EMBED_P(obj) ? -1 : (int)RARRAY(obj)->as.heap.aux.capa, + RARRAY_CONST_PTR_TRANSIENT(obj)); + } break; case T_STRING: { snprintf(buff, buff_size, "%s %s", buff, RSTRING_PTR(obj)); @@ -9954,6 +9996,7 @@ Init_GC(void) /* internal methods */ rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0); + rb_define_singleton_method(rb_mGC, "verify_transient_heap_internal_consistency", gc_verify_transient_heap_internal_consistency, 0); #if MALLOC_ALLOCATED_SIZE rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0); rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0); diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h index a05651ac3b..1512e78179 100644 --- a/include/ruby/ruby.h +++ b/include/ruby/ruby.h @@ -1020,12 +1020,15 @@ enum ruby_rarray_flags { RARRAY_EMBED_LEN_MASK = (RUBY_FL_USER4|RUBY_FL_USER3), RARRAY_EMBED_LEN_SHIFT = (RUBY_FL_USHIFT+3), + RARRAY_TRANSIENT_FLAG = RUBY_FL_USER13, + RARRAY_ENUM_END }; #define RARRAY_EMBED_FLAG (VALUE)RARRAY_EMBED_FLAG #define RARRAY_EMBED_LEN_MASK (VALUE)RARRAY_EMBED_LEN_MASK #define RARRAY_EMBED_LEN_MAX RARRAY_EMBED_LEN_MAX #define RARRAY_EMBED_LEN_SHIFT RARRAY_EMBED_LEN_SHIFT +#define RARRAY_TRANSIENT_FLAG RARRAY_TRANSIENT_FLAG struct RArray { struct RBasic basic; union { @@ -1046,9 +1049,14 @@ struct RArray { #define RARRAY_LEN(a) rb_array_len(a) #define RARRAY_LENINT(ary) rb_long2int(RARRAY_LEN(ary)) #define RARRAY_CONST_PTR(a) rb_array_const_ptr(a) +#define RARRAY_CONST_PTR_TRANSIENT(a) rb_array_const_ptr_transient(a) +#define RARRAY_TRANSIENT_P(ary) FL_TEST_RAW((ary), RARRAY_TRANSIENT_FLAG) -#define RARRAY_PTR_USE_START(a) ((VALUE *)RARRAY_CONST_PTR(a)) -#define RARRAY_PTR_USE_END(a) /* */ +VALUE *rb_ary_ptr_use_start(VALUE ary); +void rb_ary_ptr_use_end(VALUE ary); + +#define RARRAY_PTR_USE_START(a) rb_ary_ptr_use_start(a) +#define RARRAY_PTR_USE_END(a) rb_ary_ptr_use_end(a) #define RARRAY_PTR_USE(ary, ptr_name, expr) do { \ const VALUE _ary = (ary); \ @@ -1057,11 +1065,12 @@ struct RArray { RARRAY_PTR_USE_END(_ary); \ } while (0) -#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR(a)[i]) +#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR_TRANSIENT(a)[i]) #define RARRAY_ASET(a, i, v) do { \ const VALUE _ary = (a); \ + const VALUE _v = (v); \ VALUE *ptr = (VALUE *)RARRAY_PTR_USE_START(_ary); \ - RB_OBJ_WRITE(_ary, &ptr[i], (v)); \ + RB_OBJ_WRITE(_ary, &ptr[i], _v); \ RARRAY_PTR_USE_END(_ary); \ } while (0) @@ -2110,12 +2119,23 @@ rb_array_len(VALUE a) #endif static inline const VALUE * -rb_array_const_ptr(VALUE a) +rb_array_const_ptr_transient(VALUE a) { return FIX_CONST_VALUE_PTR((RBASIC(a)->flags & RARRAY_EMBED_FLAG) ? RARRAY(a)->as.ary : RARRAY(a)->as.heap.ptr); } +void rb_ary_detransient(VALUE a); + +static inline const VALUE * +rb_array_const_ptr(VALUE a) +{ + if (RARRAY_TRANSIENT_P(a)) { + rb_ary_detransient(a); + } + return rb_array_const_ptr_transient(a); +} + #if defined(EXTLIB) && defined(USE_DLN_A_OUT) /* hook for external modules */ static char *dln_libs_to_be_linked[] = { EXTLIB, 0 }; diff --git a/inits.c b/inits.c index c9687de516..7eb543104f 100644 --- a/inits.c +++ b/inits.c @@ -16,6 +16,7 @@ void rb_call_inits(void) { + CALL(TransientHeap); CALL(Method); CALL(RandomSeedCore); CALL(sym); diff --git a/insns.def b/insns.def index 132ce2f179..747e6ef8d4 100644 --- a/insns.def +++ b/insns.def @@ -524,7 +524,7 @@ newhashfromarray { VM_ASSERT(num * 2 == (rb_num_t)RARRAY_LEN(ary)); hash = rb_hash_new_with_size(num); - rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR(ary), hash); + rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR_TRANSIENT(ary), hash); } /* put new Range object.(Range.new(low, high, flag)) */ diff --git a/internal.h b/internal.h index 697a1196fa..ffe84528db 100644 --- a/internal.h +++ b/internal.h @@ -1073,6 +1073,26 @@ VALUE rb_gvar_set(struct rb_global_entry *, VALUE); VALUE rb_gvar_defined(struct rb_global_entry *); /* array.c */ + +#ifndef ARRAY_DEBUG +#define ARRAY_DEBUG 0 +#endif + +#ifdef ARRAY_DEBUG +#define RARRAY_PTR_IN_USE_FLAG FL_USER14 +#define ARY_PTR_USING_P(ary) FL_TEST_RAW((ary), RARRAY_PTR_IN_USE_FLAG) + +#else + +/* disable debug function */ +#undef RARRAY_PTR_USE_START +#undef RARRAY_PTR_USE_END +#define RARRAY_PTR_USE_START(a) ((VALUE *)RARRAY_CONST_PTR_TRANSIENT(a)) +#define RARRAY_PTR_USE_END(a) +#define ARY_PTR_USING_P(ary) 0 + +#endif + VALUE rb_ary_last(int, const VALUE *, VALUE); void rb_ary_set_len(VALUE, long); void rb_ary_delete_same(VALUE, VALUE); @@ -1100,7 +1120,7 @@ static inline VALUE rb_ary_entry_internal(VALUE ary, long offset) { long len = RARRAY_LEN(ary); - const VALUE *ptr = RARRAY_CONST_PTR(ary); + const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary); if (len == 0) return Qnil; if (offset < 0) { offset += len; @@ -1337,6 +1357,9 @@ RUBY_SYMBOL_EXPORT_END rb_wb_unprotected_newobj_of(klass, flags)) #define NEWOBJ_OF(obj,type,klass,flags) RB_NEWOBJ_OF(obj,type,klass,flags) +void *rb_aligned_malloc(size_t, size_t); +void rb_aligned_free(void *); + /* hash.c */ struct st_table *rb_hash_tbl_raw(VALUE hash); VALUE rb_hash_new_with_size(st_index_t size); diff --git a/string.c b/string.c index 3449ce6b53..48718f93ac 100644 --- a/string.c +++ b/string.c @@ -2006,8 +2006,8 @@ rb_str_format_m(VALUE str, VALUE arg) VALUE tmp = rb_check_array_type(arg); if (!NIL_P(tmp)) { - VALUE rv = rb_str_format(RARRAY_LENINT(tmp), RARRAY_CONST_PTR(tmp), str); - RB_GC_GUARD(tmp); + const long len = RARRAY_LENINT(tmp); + VALUE rv = rb_str_format(len, RARRAY_CONST_PTR(tmp), str); return rv; } return rb_str_format(1, &arg, str); diff --git a/test/ruby/test_enum.rb b/test/ruby/test_enum.rb index a4eace2d57..c56e280e06 100644 --- a/test/ruby/test_enum.rb +++ b/test/ruby/test_enum.rb @@ -1115,4 +1115,19 @@ class TestEnumerable < Test::Unit::TestCase assert_equal([1, 2, 3, 4, 5, 10], (1..100).uniq{|x| (x**2) % 10 }.first(6)) assert_equal([1, [1, 2]], Foo.new.to_enum.uniq) end + + def test_transient_heap_sort_by + klass = Class.new do + include Comparable + attr_reader :i + def initialize e + @i = e + end + def <=> other + GC.start + i <=> other.i + end + end + assert_equal [1, 2, 3, 4, 5], (1..5).sort_by{|e| klass.new e} + end end diff --git a/transient_heap.c b/transient_heap.c new file mode 100644 index 0000000000..e9a2460d88 --- /dev/null +++ b/transient_heap.c @@ -0,0 +1,834 @@ +/********************************************************************** + + transient_heap.c - implement transient_heap. + + Copyright (C) 2018 Koichi Sasada + +**********************************************************************/ + +#include "ruby/ruby.h" +#include "ruby/debug.h" +#include "vm_debug.h" +#include "gc.h" +#include "internal.h" +#include "ruby_assert.h" +#include "transient_heap.h" +#include "debug_counter.h" + +/* + * 1: enable assertions + * 2: enable verify all transient heaps + */ +#ifndef TRANSIENT_HEAP_CHECK_MODE +#define TRANSIENT_HEAP_CHECK_MODE 0 +#endif +#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr) + +/* + * 1: show events + * 2: show dump at events + * 3: show all operations + */ +#define TRANSIENT_HEAP_DEBUG 0 + +/* For Debug: Provide blocks infinitely. + * This mode generates blocks unlimitedly + * and prohibit access free'ed blocks to check invalid access. + */ +#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0 + +#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK +#include +#include +#endif + +/* For Debug: Prohibit promoting to malloc space. + */ +#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0 + +/* size configuration */ +#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024 + + /* K M */ +#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */ +#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */ +#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */ +#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE) + +#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab +#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *) + +#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1 +#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2 + +enum transient_heap_status { + transient_heap_none, + transient_heap_marking, + transient_heap_escaping +}; + +struct transient_heap_block { + struct transient_heap_block_header { + int16_t size; /* sizeof(block) = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header) */ + int16_t index; + int16_t last_marked_index; + int16_t objects; + struct transient_heap_block *next_block; + } info; + char buff[TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header)]; +}; + +struct transient_heap { + struct transient_heap_block *using_blocks; + struct transient_heap_block *marked_blocks; + struct transient_heap_block *free_blocks; + int total_objects; + int total_marked_objects; + int total_blocks; + enum transient_heap_status status; + + VALUE *promoted_objects; + int promoted_objects_size; + int promoted_objects_index; + + struct transient_heap_block *arena; + int arena_index; /* increment only */ +}; + +struct transient_alloc_header { + uint16_t magic; + uint16_t size; + int16_t next_marked_index; + int16_t dummy; + VALUE obj; +}; + +static struct transient_heap global_transient_heap; + +static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj); +static const void *transient_heap_ptr(VALUE obj, int error); +static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr); + +#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1)) + +static void +transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block) +{ + int i=0, n=0; + struct transient_alloc_header *header = NULL; + + while (iinfo.index) { + header = (void *)&block->buff[i]; + fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, header, header->size, header->next_marked_index, rb_obj_info(header->obj)); + i += header->size; + n++; + } +} + +static void +transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str) +{ + while (block) { + fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n", + type_str, block, block->info.index, block->info.objects, block->info.last_marked_index, block->info.next_block); + + transient_heap_block_dump(theap, block); + block = block->info.next_block; + } +} + +static void +transient_heap_dump(struct transient_heap* theap) +{ + fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks); + transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks"); + transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks"); + transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks"); +} + +/* Debug: dump all tarnsient_heap blocks */ +void +rb_transient_heap_dump(void) +{ + transient_heap_dump(&global_transient_heap); +} + +#if TRANSIENT_HEAP_CHECK_MODE >= 2 +static void +transient_heap_ptr_check(struct transient_heap *theap, VALUE obj) +{ + if (obj != Qundef) { + const void *ptr = transient_heap_ptr(obj, FALSE); + TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr)); + } +} + +static int +transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block) +{ + int i=0, n=0; + struct transient_alloc_header *header; + + while (iinfo.index) { + header = (void *)&block->buff[i]; + TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC); + transient_heap_ptr_check(theap, header->obj); + n ++; + i += header->size; + } + TH_ASSERT(block->info.objects == n); + + return n; +} + +static int +transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr) +{ + int n = 0; + struct transient_heap_block *block = blocks; + while (block) { + n += transient_heap_block_verify(theap, block); + *block_num_ptr += 1; + block = block->info.next_block; + } + + return n; +} +#endif + +static void +transient_heap_verify(struct transient_heap *theap) +{ +#if TRANSIENT_HEAP_CHECK_MODE >= 2 + int n=0, block_num=0; + + n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num); + n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num); + + TH_ASSERT(n == theap->total_objects); + TH_ASSERT(n >= theap->total_marked_objects); + TH_ASSERT(block_num == theap->total_blocks); +#endif +} + +/* Debug: check assertions for all transient_heap blocks */ +void +rb_transient_heap_verify(void) +{ + transient_heap_verify(&global_transient_heap); +} + +static struct transient_heap* +transient_heap_get(void) +{ + struct transient_heap* theap = &global_transient_heap; + transient_heap_verify(theap); + return theap; +} + +static void +reset_block(struct transient_heap_block *block) +{ + block->info.size = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header); + block->info.index = 0; + block->info.objects = 0; + block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; + block->info.next_block = NULL; +} + +static void +connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block) +{ + block->info.next_block = theap->free_blocks; + theap->free_blocks = block; +} + +static void +connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block) +{ + block->info.next_block = theap->using_blocks; + theap->using_blocks = block; +} + +#if 0 +static void +connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block) +{ + block->info.next_block = theap->marked_blocks; + theap->marked_blocks = block; +} +#endif + +static void +append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks) +{ + if (theap->marked_blocks) { + struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL; + while (block) { + last_block = block; + block = block->info.next_block; + } + + TH_ASSERT(last_block->info.next_block == NULL); + last_block->info.next_block = append_blocks; + } + else { + theap->marked_blocks = append_blocks; + } +} + +static struct transient_heap_block * +transient_heap_block_alloc(struct transient_heap* theap) +{ + struct transient_heap_block *block; +#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK + block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno); +#else + if (theap->arena == NULL) { + theap->arena = rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE); + } + + TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM); + block = &theap->arena[theap->arena_index++]; + TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0); +#endif + reset_block(block); + + TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0); + if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, block); + return block; +} + + +static struct transient_heap_block * +transient_heap_allocatable_block(struct transient_heap* theap) +{ + struct transient_heap_block *block; + +#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK + block = transient_heap_block_alloc(theap); + theap->total_blocks++; +#else + /* get one block from free_blocks */ + block = theap->free_blocks; + if (block) { + theap->free_blocks = block->info.next_block; + block->info.next_block = NULL; + theap->total_blocks++; + } +#endif + + return block; +} + +static struct transient_alloc_header * +transient_heap_allocatable_header(struct transient_heap* theap, size_t size) +{ + struct transient_heap_block *block = theap->using_blocks; + + while (block) { + TH_ASSERT(block->info.size >= block->info.index); + + if (block->info.size - block->info.index >= (int32_t)size) { + struct transient_alloc_header *header = (void *)&block->buff[block->info.index]; + block->info.index += size; + block->info.objects++; + return header; + } + else { + block = transient_heap_allocatable_block(theap); + if (block) connect_to_using_blocks(theap, block); + } + } + + return NULL; +} + +void * +rb_transient_heap_alloc(VALUE obj, size_t req_size) +{ + struct transient_heap* theap = transient_heap_get(); + size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN); + + TH_ASSERT(RB_TYPE_P(obj, T_ARRAY)); /* supported types */ + + if (size > TRANSIENT_HEAP_ALLOC_MAX) { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj)); + return NULL; + } +#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0 + else if (RB_OBJ_PROMOTED_RAW(obj)) { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj)); + return NULL; + } +#else + else if (RBASIC_CLASS(obj) == 0) { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj)); + return NULL; + } +#endif + else { + struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size); + if (header) { + void *ptr; + + header->size = size; + header->magic = TRANSIENT_HEAP_ALLOC_MAGIC; + header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE; + header->obj = obj; /* TODO: can we eliminate it? */ + ptr = header + 1; + + theap->total_objects++; /* statistics */ + +#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE + if (RB_OBJ_PROMOTED_RAW(obj)) { + transient_heap_promote_add(theap, obj); + } +#endif + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", header, ptr, (int)size, rb_obj_info(obj)); + + RB_DEBUG_COUNTER_INC(theap_alloc); + return ptr; + } + else { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj)); + RB_DEBUG_COUNTER_INC(theap_alloc_fail); + return NULL; + } + } +} + +void +Init_TransientHeap(void) +{ + int i, block_num; + struct transient_heap* theap = transient_heap_get(); + +#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK + block_num = 0; +#else + TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE); + block_num = TRANSIENT_HEAP_BLOCK_NUM; +#endif + for (i=0; iusing_blocks = transient_heap_allocatable_block(theap); + + theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE; + theap->promoted_objects_index = 0; + /* should not use ALLOC_N to be free from GC */ + theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size); + if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed."); +} + +static struct transient_heap_block * +blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header) +{ + struct transient_heap_block *block = blocks; + + while (block) { + if (block->buff <= (char *)header && (char *)header < block->buff + block->info.size) { + return block; + } + block = block->info.next_block; + } + + return NULL; +} + +static struct transient_heap_block * +alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header) +{ + struct transient_heap_block *block; + + if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n"); + return block; + } + else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n"); + return block; + } + else { + return NULL; + } + return block; +} + +static struct transient_alloc_header * +ptr_to_alloc_header(const void *ptr) +{ + struct transient_alloc_header *header = (void *)ptr; + header -= 1; + return header; +} + +static int +transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr) +{ + if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) { + return TRUE; + } + else { + return FALSE; + } +} + + +int +rb_transient_heap_managed_ptr_p(const void *ptr) +{ + return transient_header_managed_ptr_p(transient_heap_get(), ptr); +} + +static struct transient_heap_block * +alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header) +{ + struct transient_heap_block *block; +#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK + block = alloc_header_to_block_verbose(theap, header); + if (block == NULL) { + transient_heap_dump(theap); + rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header); + } +#else + block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1)); + TH_ASSERT(block == alloc_header_to_block_verbose(theap, header)); +#endif + return block; +} + +void +rb_transient_heap_mark(VALUE obj, const void *ptr) +{ + struct transient_alloc_header *header = ptr_to_alloc_header(ptr); + + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr); + +#if TRANSIENT_HEAP_CHECK_MODE > 0 + { + struct transient_heap* theap = transient_heap_get(); + TH_ASSERT(theap->status == transient_heap_marking); + TH_ASSERT(transient_header_managed_ptr_p(theap, ptr)); + + if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) { + transient_heap_dump(theap); + rb_bug("rb_transient_heap_mark: magic is broken"); + } + else if (header->obj != obj) { + transient_heap_dump(theap); + rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n", + rb_obj_info(header->obj), rb_obj_info(obj)); + } + } +#endif + + if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) { + /* already marked */ + return; + } + else { + struct transient_heap* theap = transient_heap_get(); + struct transient_heap_block *block = alloc_header_to_block(theap, header); + header->next_marked_index = block->info.last_marked_index; + block->info.last_marked_index = (int)((char *)header - block->buff); + theap->total_marked_objects++; + + transient_heap_verify(theap); + } +} + +static const void * +transient_heap_ptr(VALUE obj, int error) +{ + const void *ptr; + + switch (BUILTIN_TYPE(obj)) { + case T_ARRAY: + if (RARRAY_TRANSIENT_P(obj)) { + TH_ASSERT(!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG)); + ptr = RARRAY(obj)->as.heap.ptr; + } + else { + ptr = NULL; + } + break; + default: + if (error) { + rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj)); + } + else { + ptr = NULL; + } + } + + return ptr; +} + +static void +transient_heap_promote_add(struct transient_heap* theap, VALUE obj) +{ + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj)); + + if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) { + /* duplicate check */ + int i; + for (i=0; ipromoted_objects_index; i++) { + if (theap->promoted_objects[i] == obj) return; + } + } + + if (theap->promoted_objects_size <= theap->promoted_objects_index) { + theap->promoted_objects_size *= 2; + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size); + theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE)); + if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed"); + } + theap->promoted_objects[theap->promoted_objects_index++] = obj; +} + +void +rb_transient_heap_promote(VALUE obj) +{ + if (transient_heap_ptr(obj, FALSE)) { + struct transient_heap* theap = transient_heap_get(); + transient_heap_promote_add(theap, obj); + } + else { + /* ignore */ + } +} + +static struct transient_alloc_header * +alloc_header(struct transient_heap_block* block, int index) +{ + return (void *)&block->buff[index]; +} + +static void +transient_heap_reset(void) +{ + struct transient_heap* theap = transient_heap_get(); + struct transient_heap_block* block; + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n"); + + block = theap->marked_blocks; + while (block) { + struct transient_heap_block *next_block = block->info.next_block; + theap->total_objects -= block->info.objects; +#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK + if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) { + rb_bug("madvise err:%d", errno); + } + if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) { + rb_bug("mprotect err:%d", errno); + } +#else + reset_block(block); + connect_to_free_blocks(theap, block); +#endif + theap->total_blocks--; + block = next_block; + } + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks); + + theap->marked_blocks = NULL; + theap->total_marked_objects = 0; +} + +static void +transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block) +{ + int marked_index = block->info.last_marked_index; + block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; + + while (marked_index >= 0) { + struct transient_alloc_header *header = alloc_header(block, marked_index); + VALUE obj = header->obj; + + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", header, rb_obj_info(obj)); + + if (obj != Qnil) { + RB_DEBUG_COUNTER_INC(theap_evacuate); + + switch (BUILTIN_TYPE(obj)) { + case T_ARRAY: +#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE + rb_ary_transient_heap_evacuate(obj, FALSE); +#else + rb_ary_transient_heap_evacuate(obj, TRUE); +#endif + break; + default: + rb_bug("unsupporeted"); + } + header->obj = Qundef; /* for debug */ + } + marked_index = header->next_marked_index; + } +} + +static void +transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status) +{ + TH_ASSERT(theap->status != status); + theap->status = status; +} + +static void +transient_heap_evacuate(void *dmy) +{ + struct transient_heap* theap = transient_heap_get(); + + if (theap->status == transient_heap_marking) { + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n"); + } + else { + VALUE gc_disabled = rb_gc_disable(); + struct transient_heap_block* block; + + if (TRANSIENT_HEAP_DEBUG >= 1) { + int i; + fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks); + if (TRANSIENT_HEAP_DEBUG >= 4) { + for (i=0; ipromoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i])); + } + } + if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); + + TH_ASSERT(theap->status == transient_heap_none); + transient_heap_update_status(theap, transient_heap_escaping); + + /* evacuate from marked blocks */ + block = theap->marked_blocks; + while (block) { + transient_heap_block_evacuate(theap, block); + block = block->info.next_block; + } + + /* evacuate from using blocks + only affect incremental marking */ + block = theap->using_blocks; + while (block) { + transient_heap_block_evacuate(theap, block); + block = block->info.next_block; + } + + /* all objects in marked_objects are escaped. */ + transient_heap_reset(); + + if (TRANSIENT_HEAP_DEBUG > 0) { + fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks); + } + + transient_heap_verify(theap); + transient_heap_update_status(theap, transient_heap_none); + if (gc_disabled != Qtrue) rb_gc_enable(); + } +} + +static void +clear_marked_index(struct transient_heap_block* block) +{ + int marked_index = block->info.last_marked_index; + + while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) { + struct transient_alloc_header *header = alloc_header(block, marked_index); + TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE); + if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", block, marked_index); + + marked_index = header->next_marked_index; + header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE; + } + + block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; +} + +static void +blocks_clear_marked_index(struct transient_heap_block* block) +{ + while (block) { + clear_marked_index(block); + block = block->info.next_block; + } +} + +void +rb_transient_heap_start_marking(int full_marking) +{ + struct transient_heap* theap = transient_heap_get(); + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promtoed:%d full_marking:%d\n", + theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking); + if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); + + blocks_clear_marked_index(theap->marked_blocks); + blocks_clear_marked_index(theap->using_blocks); + + if (theap->using_blocks) { + if (theap->using_blocks->info.objects > 0) { + append_to_marked_blocks(theap, theap->using_blocks); + theap->using_blocks = NULL; + } + else { + append_to_marked_blocks(theap, theap->using_blocks->info.next_block); + theap->using_blocks->info.next_block = NULL; + } + } + + if (theap->using_blocks == NULL) { + theap->using_blocks = transient_heap_allocatable_block(theap); + } + + TH_ASSERT(theap->status == transient_heap_none); + transient_heap_update_status(theap, transient_heap_marking); + theap->total_marked_objects = 0; + + if (full_marking) { + theap->promoted_objects_index = 0; + } + else { /* mark promoted objects */ + int i; + for (i=0; ipromoted_objects_index; i++) { + VALUE obj = theap->promoted_objects[i]; + const void *ptr = transient_heap_ptr(obj, TRUE); + if (ptr) { + rb_transient_heap_mark(obj, ptr); + } + } + } + + transient_heap_verify(theap); +} + +void +rb_transient_heap_finish_marking(void) +{ + struct transient_heap* theap = transient_heap_get(); + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n", + theap->total_objects, + theap->total_marked_objects); + if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); + + TH_ASSERT(theap->total_objects >= theap->total_marked_objects); + + TH_ASSERT(theap->status == transient_heap_marking); + transient_heap_update_status(theap, transient_heap_none); + + if (theap->total_marked_objects > 0) { + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n"); + rb_postponed_job_register_one(0, transient_heap_evacuate, NULL); + } + else { + transient_heap_reset(); + } + + transient_heap_verify(theap); +} diff --git a/transient_heap.h b/transient_heap.h new file mode 100644 index 0000000000..ac0eaadb78 --- /dev/null +++ b/transient_heap.h @@ -0,0 +1,38 @@ +/********************************************************************** + + transient_heap.h - declarations of transient_heap related APIs. + + Copyright (C) 2018 Koichi Sasada + +**********************************************************************/ + +#ifndef RUBY_TRANSIENT_HEAP_H +#define RUBY_TRANSIENT_HEAP_H + +/* public API */ + +/* Allocate req_size bytes from transient_heap. + Allocated memories are free-ed when next GC + if this memory is not marked by `rb_transient_heap_mark()`. + */ +void *rb_transient_heap_alloc(VALUE obj, size_t req_size); + +/* If `obj` uses a memory pointed by `ptr` from transient_heap, + you need to call `rb_transient_heap_mark(obj, ptr)` + to assert liveness of `obj` (and ptr). */ +void rb_transient_heap_mark(VALUE obj, const void *ptr); + +/* used by gc.c */ +void rb_transient_heap_promote(VALUE obj); +void rb_transient_heap_start_marking(int full_marking); +void rb_transient_heap_finish_marking(void); + +/* for debug API */ +void rb_transient_heap_dump(void); +void rb_transient_heap_verify(void); +int rb_transient_heap_managed_ptr_p(const void *ptr); + +/* evacuate functions */ +void rb_ary_transient_heap_evacuate(VALUE ary, int promote); + +#endif diff --git a/vm_args.c b/vm_args.c index f9f74b5b0b..16284c0f33 100644 --- a/vm_args.c +++ b/vm_args.c @@ -164,7 +164,7 @@ args_copy(struct args_info *args) static inline const VALUE * args_rest_argv(struct args_info *args) { - return RARRAY_CONST_PTR(args->rest) + args->rest_index; + return RARRAY_CONST_PTR_TRANSIENT(args->rest) + args->rest_index; } static inline VALUE @@ -314,7 +314,7 @@ args_setup_post_parameters(struct args_info *args, int argc, VALUE *locals) { long len; len = RARRAY_LEN(args->rest); - MEMCPY(locals, RARRAY_CONST_PTR(args->rest) + len - argc, VALUE, argc); + MEMCPY(locals, RARRAY_CONST_PTR_TRANSIENT(args->rest) + len - argc, VALUE, argc); rb_ary_resize(args->rest, len - argc); } @@ -334,13 +334,13 @@ args_setup_opt_parameters(struct args_info *args, int opt_max, VALUE *locals) args->argc = 0; if (args->rest) { - int len = RARRAY_LENINT(args->rest); - const VALUE *argv = RARRAY_CONST_PTR(args->rest); + int len = RARRAY_LENINT(args->rest); + const VALUE *argv = RARRAY_CONST_PTR_TRANSIENT(args->rest); - for (; irest_index < len; i++, args->rest_index++) { - locals[i] = argv[args->rest_index]; - } - } + for (; irest_index < len; i++, args->rest_index++) { + locals[i] = argv[args->rest_index]; + } + } /* initialize by nil */ for (j=i; jsp--; if (!NIL_P(ary)) { - const VALUE *ptr = RARRAY_CONST_PTR(ary); - long len = RARRAY_LEN(ary), i; + const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary); + long len = RARRAY_LEN(ary), i; - CHECK_VM_STACK_OVERFLOW(cfp, len); + CHECK_VM_STACK_OVERFLOW(cfp, len); - for (i = 0; i < len; i++) { - *cfp->sp++ = ptr[i]; - } - calling->argc += i - 1; + for (i = 0; i < len; i++) { + *cfp->sp++ = ptr[i]; + } + calling->argc += i - 1; } } diff --git a/vm_eval.c b/vm_eval.c index 1176c494e1..a864b75712 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -764,7 +764,7 @@ rb_apply(VALUE recv, ID mid, VALUE args) return ret; } argv = ALLOCA_N(VALUE, argc); - MEMCPY(argv, RARRAY_CONST_PTR(args), VALUE, argc); + MEMCPY(argv, RARRAY_CONST_PTR_TRANSIENT(args), VALUE, argc); return rb_call(recv, mid, argc, argv, CALL_FCALL); } diff --git a/vm_insnhelper.c b/vm_insnhelper.c index d16c318f2c..5904ce7233 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1265,7 +1265,7 @@ vm_expandarray(VALUE *sp, VALUE ary, rb_num_t num, int flag) len = 1; } else { - ptr = RARRAY_CONST_PTR(ary); + ptr = RARRAY_CONST_PTR_TRANSIENT(ary); len = (rb_num_t)RARRAY_LEN(ary); }