Closes [Feature #19729]

Previously 2 bits of the flags on each RVALUE are reserved to store the
number of GC cycles that each object has survived. This commit
introduces a new bit array on the heap page, called age_bits, to store
that information instead.

This patch still reserves one of the age bits in the flags (the old
FL_PROMOTED0 bit, now renamed FL_PROMOTED).

This is set to 0 for young objects and 1 for old objects, and is used as
a performance optimisation for the write barrier. Fetching the age_bits
from the heap page and doing the required math to calculate if the
object was old or not would slow down the write barrier. So we keep this
bit synced in the flags for fast access.
This commit is contained in:
Matt Valentine-House 2023-03-29 21:05:13 +01:00
Родитель 7524675330
Коммит d426343418
6 изменённых файлов: 86 добавлений и 112 удалений

Просмотреть файл

@ -232,7 +232,6 @@ class_alloc(VALUE flags, VALUE klass)
size_t alloc_size = sizeof(struct RClass) + sizeof(rb_classext_t);
flags &= T_MASK;
flags |= FL_PROMOTED1 /* start from age == 2 */;
if (RGENGC_WB_PROTECTED_CLASS) flags |= FL_WB_PROTECTED;
NEWOBJ_OF(obj, struct RClass, klass, flags, alloc_size, 0);

134
gc.c
Просмотреть файл

@ -939,6 +939,9 @@ static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
static bool heap_page_alloc_use_mmap;
#endif
#define RVALUE_AGE_BIT_COUNT 2
#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
struct heap_page {
short slot_size;
short total_slots;
@ -968,6 +971,7 @@ struct heap_page {
/* If set, the object is not movable */
bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
};
/*
@ -1011,6 +1015,35 @@ asan_unlock_freelist(struct heap_page *page)
#define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3
#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT))
#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT)
#define RVALUE_OLD_AGE 3
static int
RVALUE_AGE_GET(VALUE obj)
{
bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK;
}
static void
RVALUE_AGE_SET(VALUE obj, int age)
{
RUBY_ASSERT(age <= RVALUE_OLD_AGE);
bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
// clear the bits
age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj)));
// shift the correct value in
age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj));
if (age == RVALUE_OLD_AGE) {
RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED);
}
else {
RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED);
}
}
/* Aliases */
#define rb_objspace (*rb_objspace_of(GET_VM()))
#define rb_objspace_of(vm) ((vm)->objspace)
@ -1485,8 +1518,6 @@ asan_poison_object_restore(VALUE obj, void *ptr)
#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
#define RVALUE_OLD_AGE 3
#define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */
static int rgengc_remembered(rb_objspace_t *objspace, VALUE obj);
static int rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj);
@ -1494,12 +1525,6 @@ static int rgengc_remember(rb_objspace_t *objspace, VALUE obj);
static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap);
static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
static inline int
RVALUE_FLAGS_AGE(VALUE flags)
{
return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT);
}
static int
check_rvalue_consistency_force(const VALUE obj, int terminate)
{
@ -1539,7 +1564,7 @@ check_rvalue_consistency_force(const VALUE obj, int terminate)
const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0;
const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
const int age = RVALUE_AGE_GET((VALUE)obj);
if (GET_HEAP_PAGE(obj)->flags.in_tomb) {
fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", obj_info(obj));
@ -1682,29 +1707,16 @@ RVALUE_UNCOLLECTIBLE(VALUE obj)
return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
}
static inline int
RVALUE_OLD_P_RAW(VALUE obj)
{
const VALUE promoted = FL_PROMOTED0 | FL_PROMOTED1;
return (RBASIC(obj)->flags & promoted) == promoted;
}
static inline int
RVALUE_OLD_P(VALUE obj)
{
GC_ASSERT(!RB_SPECIAL_CONST_P(obj));
check_rvalue_consistency(obj);
return RVALUE_OLD_P_RAW(obj);
// Because this will only ever be called on GC controlled objects,
// we can use the faster _RAW function here
return RB_OBJ_PROMOTED_RAW(obj);
}
#if RGENGC_CHECK_MODE || GC_DEBUG
static inline int
RVALUE_AGE(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
}
#endif
static inline void
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
@ -1725,39 +1737,39 @@ RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
}
static inline VALUE
RVALUE_FLAGS_AGE_SET(VALUE flags, int age)
{
flags &= ~(FL_PROMOTED0 | FL_PROMOTED1);
flags |= (age << RVALUE_AGE_SHIFT);
return flags;
}
/* set age to age+1 */
static inline void
RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
{
VALUE flags = RBASIC(obj)->flags;
int age = RVALUE_FLAGS_AGE(flags);
int age = RVALUE_AGE_GET((VALUE)obj);
if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj));
}
age++;
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(flags, age);
RVALUE_AGE_SET(obj, age);
if (age == RVALUE_OLD_AGE) {
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
}
check_rvalue_consistency(obj);
}
static inline void
RVALUE_DEMOTE_RAW(rb_objspace_t *objspace, VALUE obj)
RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
check_rvalue_consistency(obj);
}
static inline void
RVALUE_AGE_RESET(VALUE obj)
{
RVALUE_AGE_SET(obj, 0);
}
static inline void
@ -1770,7 +1782,8 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj);
}
RVALUE_DEMOTE_RAW(objspace, obj);
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
RVALUE_AGE_RESET(obj);
if (RVALUE_MARKED(obj)) {
objspace->rgengc.old_objects--;
@ -1779,22 +1792,6 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
check_rvalue_consistency(obj);
}
static inline void
RVALUE_AGE_RESET_RAW(VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
}
static inline void
RVALUE_AGE_RESET(VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RVALUE_AGE_RESET_RAW(obj);
check_rvalue_consistency(obj);
}
static inline int
RVALUE_BLACK_P(VALUE obj)
{
@ -1962,6 +1959,8 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj
page->freelist = p;
asan_lock_freelist(page);
RVALUE_AGE_RESET(obj);
if (RGENGC_CHECK_MODE &&
/* obj should belong to page */
!(page->start <= (uintptr_t)obj &&
@ -2505,6 +2504,11 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
p->as.basic.flags = flags;
*((VALUE *)&p->as.basic.klass) = klass;
int t = flags & RUBY_T_MASK;
if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
RVALUE_AGE_SET_CANDIDATE(objspace, obj);
}
#if RACTOR_CHECK_MODE
rb_ractor_setup_belonging(obj);
#endif
@ -2521,12 +2525,6 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
GC_ASSERT(RVALUE_OLD_P(obj) == FALSE);
GC_ASSERT(RVALUE_WB_UNPROTECTED(obj) == FALSE);
if (flags & FL_PROMOTED1) {
if (RVALUE_AGE(obj) != 2) rb_bug("newobj: %s of age (%d) != 2.", obj_info(obj), RVALUE_AGE(obj));
}
else {
if (RVALUE_AGE(obj) > 0) rb_bug("newobj: %s of age (%d) > 0.", obj_info(obj), RVALUE_AGE(obj));
}
if (rgengc_remembered(objspace, (VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj));
}
RB_VM_LOCK_LEAVE_NO_BARRIER();
@ -9017,7 +9015,7 @@ rb_copy_wb_protected_attribute(VALUE dest, VALUE obj)
if (RVALUE_WB_UNPROTECTED(obj) && !RVALUE_WB_UNPROTECTED(dest)) {
if (!RVALUE_OLD_P(dest)) {
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
RVALUE_AGE_RESET_RAW(dest);
RVALUE_AGE_RESET(dest);
}
else {
RVALUE_DEMOTE(objspace, dest);
@ -9789,6 +9787,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s
int marked;
int wb_unprotected;
int uncollectible;
int age;
RVALUE *dest = (RVALUE *)free;
RVALUE *src = (RVALUE *)scan;
@ -9804,6 +9803,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s
wb_unprotected = RVALUE_WB_UNPROTECTED((VALUE)src);
uncollectible = RVALUE_UNCOLLECTIBLE((VALUE)src);
bool remembered = RVALUE_REMEMBERED((VALUE)src);
age = RVALUE_AGE_GET((VALUE)src);
/* Clear bits for eventual T_MOVED */
CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)src), (VALUE)src);
@ -9846,6 +9846,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s
}
memset(src, 0, src_slot_size);
RVALUE_AGE_RESET((VALUE)src);
/* Set bits for object in new location */
if (remembered) {
@ -9876,6 +9877,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)dest), (VALUE)dest);
}
RVALUE_AGE_SET((VALUE)dest, age);
/* Assign forwarding address */
src->as.moved.flags = T_MOVED;
src->as.moved.dummy = Qundef;
@ -13439,7 +13441,7 @@ rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj
}
}
else {
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
const int age = RVALUE_AGE_GET(obj);
if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
APPEND_F("%p [%d%s%s%s%s%s%s] %s ",
@ -13776,7 +13778,7 @@ rb_gcdebug_print_obj_condition(VALUE obj)
fprintf(stderr, "marked? : %s\n", MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) ? "true" : "false");
fprintf(stderr, "pinned? : %s\n", MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) ? "true" : "false");
fprintf(stderr, "age? : %d\n", RVALUE_AGE(obj));
fprintf(stderr, "age? : %d\n", RVALUE_AGE_GET(obj));
fprintf(stderr, "old? : %s\n", RVALUE_OLD_P(obj) ? "true" : "false");
fprintf(stderr, "WB-protected?: %s\n", RVALUE_WB_UNPROTECTED(obj) ? "false" : "true");
fprintf(stderr, "remembered? : %s\n", RVALUE_REMEMBERED(obj) ? "true" : "false");

Просмотреть файл

@ -173,10 +173,9 @@ rbimpl_typeddata_flags {
RUBY_TYPED_WB_PROTECTED = RUBY_FL_WB_PROTECTED, /* THIS FLAG DEPENDS ON Ruby version */
/**
* This flag is mysterious. It seems nobody is currently using it. The
* intention of this flag is also unclear. We need further investigations.
* This flag no longer in use
*/
RUBY_TYPED_PROMOTED1 = RUBY_FL_PROMOTED1, /* THIS FLAG DEPENDS ON Ruby version */
RUBY_TYPED_UNUSED = RUBY_FL_UNUSED6,
/**
* This flag determines whether marking and compaction should be carried out

Просмотреть файл

@ -57,8 +57,7 @@
#define FL_SINGLETON RBIMPL_CAST((VALUE)RUBY_FL_SINGLETON) /**< @old{RUBY_FL_SINGLETON} */
#define FL_WB_PROTECTED RBIMPL_CAST((VALUE)RUBY_FL_WB_PROTECTED) /**< @old{RUBY_FL_WB_PROTECTED} */
#define FL_PROMOTED0 RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED0) /**< @old{RUBY_FL_PROMOTED0} */
#define FL_PROMOTED1 RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED1) /**< @old{RUBY_FL_PROMOTED1} */
#define FL_PROMOTED RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED) /**< @old{RUBY_FL_PROMOTED} */
#define FL_FINALIZE RBIMPL_CAST((VALUE)RUBY_FL_FINALIZE) /**< @old{RUBY_FL_FINALIZE} */
#define FL_TAINT RBIMPL_CAST((VALUE)RUBY_FL_TAINT) /**< @old{RUBY_FL_TAINT} */
#define FL_SHAREABLE RBIMPL_CAST((VALUE)RUBY_FL_SHAREABLE) /**< @old{RUBY_FL_SHAREABLE} */
@ -200,12 +199,15 @@ ruby_fl_type {
RUBY_FL_WB_PROTECTED = (1<<5),
/**
* This flag has something to do with our garbage collector. These days
* ruby objects are "generational". There are those who are young and
* those who are old. Young objects are prone to die; monitored relatively
* extensively by the garbage collector. OTOH old objects tend to live
* longer. They are relatively rarely considered. This flag is set when a
* object experienced promotion i.e. survived a garbage collection.
* Ruby objects are "generational". There are young objects & old objects.
* Young objects are prone to die & monitored relatively extensively by the
* garbage collector. Old objects tend to live longer & are monitored less
* frequently. When an object survives a GC, its age is incremented. When
* age is equal to RVALUE_OLD_AGE, the object becomes Old. This flag is set
* when an object becomes old, and is used by the write barrier to check if
* an old object should be considered for marking more frequently - as old
* objects that have references added between major GCs need to be remarked
* to prevent the referred object being mistakenly swept.
*
* @internal
*
@ -213,41 +215,14 @@ ruby_fl_type {
* 3rd parties. It must be an implementation detail that they should never
* know. Might better be hidden.
*/
RUBY_FL_PROMOTED0 = (1<<5),
RUBY_FL_PROMOTED = (1<<5),
/**
* This flag has something to do with our garbage collector. These days
* ruby objects are "generational". There are those who are young and
* those who are old. Young objects are prone to die; monitored relatively
* extensively by the garbage collector. OTOH old objects tend to live
* longer. They are relatively rarely considered. This flag is set when a
* object experienced two promotions i.e. survived garbage collections
* twice.
* This flag is no longer in use
*
* @internal
*
* But honestly, @shyouhei doesn't think this flag should be visible from
* 3rd parties. It must be an implementation detail that they should never
* know. Might better be hidden.
*/
RUBY_FL_PROMOTED1 = (1<<6),
/**
* This flag has something to do with our garbage collector. These days
* ruby objects are "generational". There are those who are young and
* those who are old. Young objects are prone to die; monitored relatively
* extensively by the garbage collector. OTOH old objects tend to live
* longer. They are relatively rarely considered. This flag is set when a
* object experienced promotions i.e. survived more than one garbage
* collections.
*
* @internal
*
* But honestly, @shyouhei doesn't think this flag should be visible from
* 3rd parties. It must be an implementation detail that they should never
* know. Might better be hidden.
*/
RUBY_FL_PROMOTED = RUBY_FL_PROMOTED0 | RUBY_FL_PROMOTED1,
RUBY_FL_UNUSED6 = (1<<6),
/**
* This flag has something to do with finalisers. A ruby object can have

2
proc.c
Просмотреть файл

@ -70,7 +70,7 @@ CLONESETUP(VALUE clone, VALUE obj)
RBIMPL_ASSERT_OR_ASSUME(! RB_SPECIAL_CONST_P(obj));
RBIMPL_ASSERT_OR_ASSUME(! RB_SPECIAL_CONST_P(clone));
const VALUE flags = RUBY_FL_PROMOTED0 | RUBY_FL_PROMOTED1 | RUBY_FL_FINALIZE;
const VALUE flags = RUBY_FL_PROMOTED | RUBY_FL_FINALIZE;
rb_obj_setup(clone, rb_singleton_class_clone(obj),
RB_FL_TEST_RAW(obj, ~flags));
rb_singleton_class_attached(RBASIC_CLASS(clone), clone);

Просмотреть файл

@ -192,9 +192,8 @@ pub type ruby_value_type = u32;
pub const RUBY_FL_USHIFT: ruby_fl_ushift = 12;
pub type ruby_fl_ushift = u32;
pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32;
pub const RUBY_FL_PROMOTED0: ruby_fl_type = 32;
pub const RUBY_FL_PROMOTED1: ruby_fl_type = 64;
pub const RUBY_FL_PROMOTED: ruby_fl_type = 96;
pub const RUBY_FL_PROMOTED: ruby_fl_type = 32;
pub const RUBY_FL_UNUSED6: ruby_fl_type = 64;
pub const RUBY_FL_FINALIZE: ruby_fl_type = 128;
pub const RUBY_FL_TAINT: ruby_fl_type = 0;
pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256;