2022-10-03 18:14:32 +03:00
|
|
|
#ifndef RUBY_SHAPE_H
|
|
|
|
#define RUBY_SHAPE_H
|
|
|
|
#if (SIZEOF_UINT64_T == SIZEOF_VALUE)
|
|
|
|
#define SIZEOF_SHAPE_T 4
|
|
|
|
#define SHAPE_IN_BASIC_FLAGS 1
|
|
|
|
typedef uint32_t attr_index_t;
|
|
|
|
#else
|
|
|
|
#define SIZEOF_SHAPE_T 2
|
|
|
|
#define SHAPE_IN_BASIC_FLAGS 0
|
|
|
|
typedef uint16_t attr_index_t;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define MAX_IVARS (attr_index_t)(-1)
|
|
|
|
|
|
|
|
#if RUBY_DEBUG || (defined(VM_CHECK_MODE) && VM_CHECK_MODE > 0)
|
|
|
|
# if SIZEOF_SHAPE_T == 4
|
|
|
|
typedef uint32_t shape_id_t;
|
2022-11-18 21:29:41 +03:00
|
|
|
# define SHAPE_ID_NUM_BITS 16
|
2022-10-03 18:14:32 +03:00
|
|
|
# else
|
|
|
|
typedef uint16_t shape_id_t;
|
2022-11-18 21:29:41 +03:00
|
|
|
# define SHAPE_ID_NUM_BITS 16
|
2022-10-03 18:14:32 +03:00
|
|
|
# endif
|
|
|
|
#else
|
|
|
|
# if SIZEOF_SHAPE_T == 4
|
|
|
|
typedef uint32_t shape_id_t;
|
2022-11-18 21:29:41 +03:00
|
|
|
# define SHAPE_ID_NUM_BITS 32
|
2022-10-03 18:14:32 +03:00
|
|
|
# else
|
|
|
|
typedef uint16_t shape_id_t;
|
2022-11-18 21:29:41 +03:00
|
|
|
# define SHAPE_ID_NUM_BITS 16
|
2022-10-03 18:14:32 +03:00
|
|
|
# endif
|
|
|
|
#endif
|
|
|
|
|
2022-11-18 21:29:41 +03:00
|
|
|
# define SHAPE_MASK (((uintptr_t)1 << SHAPE_ID_NUM_BITS) - 1)
|
|
|
|
# define SHAPE_FLAG_MASK (((VALUE)-1) >> SHAPE_ID_NUM_BITS)
|
2022-10-03 18:14:32 +03:00
|
|
|
|
2022-11-18 21:29:41 +03:00
|
|
|
# define SHAPE_FLAG_SHIFT ((SIZEOF_VALUE * 8) - SHAPE_ID_NUM_BITS)
|
2022-10-03 18:14:32 +03:00
|
|
|
|
|
|
|
# define SHAPE_BITMAP_SIZE 16384
|
|
|
|
|
|
|
|
# define MAX_SHAPE_ID (SHAPE_MASK - 1)
|
|
|
|
# define INVALID_SHAPE_ID SHAPE_MASK
|
|
|
|
# define ROOT_SHAPE_ID 0x0
|
2022-11-08 23:35:31 +03:00
|
|
|
// We use SIZE_POOL_COUNT number of shape IDs for transitions out of different size pools
|
|
|
|
// The next available shapd ID will be the SPECIAL_CONST_SHAPE_ID
|
2022-11-18 02:57:11 +03:00
|
|
|
# define SPECIAL_CONST_SHAPE_ID (SIZE_POOL_COUNT * 2)
|
2022-10-03 18:14:32 +03:00
|
|
|
|
|
|
|
struct rb_shape {
|
|
|
|
struct rb_id_table * edges; // id_table from ID (ivar) to next shape
|
|
|
|
ID edge_name; // ID (ivar) for transition from parent to rb_shape
|
2022-10-21 23:24:29 +03:00
|
|
|
attr_index_t next_iv_index;
|
2022-11-08 23:35:31 +03:00
|
|
|
uint32_t capacity; // Total capacity of the object with this shape
|
2022-10-03 18:14:32 +03:00
|
|
|
uint8_t type;
|
2022-11-08 23:35:31 +03:00
|
|
|
uint8_t size_pool_index;
|
2022-10-03 20:52:40 +03:00
|
|
|
shape_id_t parent_id;
|
2022-10-03 18:14:32 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
typedef struct rb_shape rb_shape_t;
|
|
|
|
|
|
|
|
enum shape_type {
|
|
|
|
SHAPE_ROOT,
|
|
|
|
SHAPE_IVAR,
|
|
|
|
SHAPE_FROZEN,
|
2022-11-08 23:35:31 +03:00
|
|
|
SHAPE_CAPACITY_CHANGE,
|
2022-10-03 18:14:32 +03:00
|
|
|
SHAPE_IVAR_UNDEF,
|
2022-11-08 23:35:31 +03:00
|
|
|
SHAPE_INITIAL_CAPACITY,
|
2022-11-18 02:57:11 +03:00
|
|
|
SHAPE_T_OBJECT,
|
2022-10-03 18:14:32 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
#if SHAPE_IN_BASIC_FLAGS
|
|
|
|
static inline shape_id_t
|
|
|
|
RBASIC_SHAPE_ID(VALUE obj)
|
|
|
|
{
|
|
|
|
RUBY_ASSERT(!RB_SPECIAL_CONST_P(obj));
|
|
|
|
return (shape_id_t)(SHAPE_MASK & ((RBASIC(obj)->flags) >> SHAPE_FLAG_SHIFT));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
RBASIC_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id)
|
|
|
|
{
|
|
|
|
// Ractors are occupying the upper 32 bits of flags, but only in debug mode
|
|
|
|
// Object shapes are occupying top bits
|
|
|
|
RBASIC(obj)->flags &= SHAPE_FLAG_MASK;
|
|
|
|
RBASIC(obj)->flags |= ((VALUE)(shape_id) << SHAPE_FLAG_SHIFT);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline shape_id_t
|
|
|
|
ROBJECT_SHAPE_ID(VALUE obj)
|
|
|
|
{
|
|
|
|
RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
|
|
|
|
return RBASIC_SHAPE_ID(obj);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
ROBJECT_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id)
|
|
|
|
{
|
|
|
|
RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
|
|
|
|
RBASIC_SET_SHAPE_ID(obj, shape_id);
|
|
|
|
}
|
|
|
|
|
2022-11-01 00:05:37 +03:00
|
|
|
static inline shape_id_t
|
|
|
|
RCLASS_SHAPE_ID(VALUE obj)
|
|
|
|
{
|
|
|
|
RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE));
|
|
|
|
return RBASIC_SHAPE_ID(obj);
|
|
|
|
}
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
#else
|
|
|
|
|
|
|
|
static inline shape_id_t
|
|
|
|
ROBJECT_SHAPE_ID(VALUE obj)
|
|
|
|
{
|
|
|
|
RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
|
|
|
|
return (shape_id_t)(SHAPE_MASK & (RBASIC(obj)->flags >> SHAPE_FLAG_SHIFT));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
ROBJECT_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id)
|
|
|
|
{
|
|
|
|
RBASIC(obj)->flags &= SHAPE_FLAG_MASK;
|
|
|
|
RBASIC(obj)->flags |= ((VALUE)(shape_id) << SHAPE_FLAG_SHIFT);
|
|
|
|
}
|
2022-11-01 00:05:37 +03:00
|
|
|
|
|
|
|
MJIT_SYMBOL_EXPORT_BEGIN
|
|
|
|
shape_id_t rb_rclass_shape_id(VALUE obj);
|
|
|
|
MJIT_SYMBOL_EXPORT_END
|
|
|
|
|
|
|
|
static inline shape_id_t RCLASS_SHAPE_ID(VALUE obj) {
|
|
|
|
return rb_rclass_shape_id(obj);
|
|
|
|
}
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
#endif
|
|
|
|
|
|
|
|
bool rb_shape_root_shape_p(rb_shape_t* shape);
|
2022-11-08 23:35:31 +03:00
|
|
|
rb_shape_t * rb_shape_get_root_shape(void);
|
32 bit comparison on shape id
This commit changes the shape id comparisons to use a 32 bit comparison
rather than 64 bit. That means we don't need to load the shape id to a
register on x86 machines.
Given the following program:
```ruby
class Foo
def initialize
@foo = 1
@bar = 1
end
def read
[@foo, @bar]
end
end
foo = Foo.new
foo.read
foo.read
foo.read
foo.read
foo.read
puts RubyVM::YJIT.disasm(Foo.instance_method(:read))
```
The machine code we generated _before_ this change is like this:
```
== BLOCK 1/4, ISEQ RANGE [0,3), 65 bytes ======================
# getinstancevariable
0x559a18623023: mov rax, qword ptr [r13 + 0x18]
# guard object is heap
0x559a18623027: test al, 7
0x559a1862302a: jne 0x559a1862502d
0x559a18623030: cmp rax, 4
0x559a18623034: jbe 0x559a1862502d
# guard shape, embedded, and T_OBJECT
0x559a1862303a: mov rcx, qword ptr [rax]
0x559a1862303d: movabs r11, 0xffff00000000201f
0x559a18623047: and rcx, r11
0x559a1862304a: movabs r11, 0xb000000002001
0x559a18623054: cmp rcx, r11
0x559a18623057: jne 0x559a18625046
0x559a1862305d: mov rax, qword ptr [rax + 0x18]
0x559a18623061: mov qword ptr [rbx], rax
== BLOCK 2/4, ISEQ RANGE [3,6), 0 bytes =======================
== BLOCK 3/4, ISEQ RANGE [3,6), 47 bytes ======================
# gen_direct_jmp: fallthrough
# getinstancevariable
# regenerate_branch
# getinstancevariable
# regenerate_branch
0x559a18623064: mov rax, qword ptr [r13 + 0x18]
# guard shape, embedded, and T_OBJECT
0x559a18623068: mov rcx, qword ptr [rax]
0x559a1862306b: movabs r11, 0xffff00000000201f
0x559a18623075: and rcx, r11
0x559a18623078: movabs r11, 0xb000000002001
0x559a18623082: cmp rcx, r11
0x559a18623085: jne 0x559a18625099
0x559a1862308b: mov rax, qword ptr [rax + 0x20]
0x559a1862308f: mov qword ptr [rbx + 8], rax
```
After this change, it's like this:
```
== BLOCK 1/4, ISEQ RANGE [0,3), 41 bytes ======================
# getinstancevariable
0x5560c986d023: mov rax, qword ptr [r13 + 0x18]
# guard object is heap
0x5560c986d027: test al, 7
0x5560c986d02a: jne 0x5560c986f02d
0x5560c986d030: cmp rax, 4
0x5560c986d034: jbe 0x5560c986f02d
# guard shape
0x5560c986d03a: cmp word ptr [rax + 6], 0x19
0x5560c986d03f: jne 0x5560c986f046
0x5560c986d045: mov rax, qword ptr [rax + 0x10]
0x5560c986d049: mov qword ptr [rbx], rax
== BLOCK 2/4, ISEQ RANGE [3,6), 0 bytes =======================
== BLOCK 3/4, ISEQ RANGE [3,6), 23 bytes ======================
# gen_direct_jmp: fallthrough
# getinstancevariable
# regenerate_branch
# getinstancevariable
# regenerate_branch
0x5560c986d04c: mov rax, qword ptr [r13 + 0x18]
# guard shape
0x5560c986d050: cmp word ptr [rax + 6], 0x19
0x5560c986d055: jne 0x5560c986f099
0x5560c986d05b: mov rax, qword ptr [rax + 0x18]
0x5560c986d05f: mov qword ptr [rbx + 8], rax
```
The first ivar read is a bit more complex, but the second ivar read is
much simpler. I think eventually we could teach the context about the
shape, then emit only one shape guard.
2022-11-15 22:46:18 +03:00
|
|
|
uint8_t rb_shape_id_num_bits(void);
|
2022-10-03 18:14:32 +03:00
|
|
|
|
|
|
|
rb_shape_t* rb_shape_get_shape_by_id_without_assertion(shape_id_t shape_id);
|
2022-11-10 19:36:24 +03:00
|
|
|
rb_shape_t * rb_shape_get_parent(rb_shape_t * shape);
|
2022-10-03 18:14:32 +03:00
|
|
|
|
|
|
|
MJIT_SYMBOL_EXPORT_BEGIN
|
|
|
|
rb_shape_t* rb_shape_get_shape_by_id(shape_id_t shape_id);
|
|
|
|
void rb_shape_set_shape(VALUE obj, rb_shape_t* shape);
|
|
|
|
shape_id_t rb_shape_get_shape_id(VALUE obj);
|
|
|
|
rb_shape_t* rb_shape_get_shape(VALUE obj);
|
|
|
|
int rb_shape_frozen_shape_p(rb_shape_t* shape);
|
|
|
|
void rb_shape_transition_shape_frozen(VALUE obj);
|
|
|
|
void rb_shape_transition_shape_remove_ivar(VALUE obj, ID id, rb_shape_t *shape);
|
2022-11-08 23:35:31 +03:00
|
|
|
rb_shape_t * rb_shape_transition_shape_capa(rb_shape_t * shape, uint32_t new_capacity);
|
|
|
|
rb_shape_t * rb_shape_get_next_iv_shape(rb_shape_t * shape, ID id);
|
2022-10-03 18:14:32 +03:00
|
|
|
rb_shape_t* rb_shape_get_next(rb_shape_t* shape, VALUE obj, ID id);
|
|
|
|
bool rb_shape_get_iv_index(rb_shape_t * shape, ID id, attr_index_t * value);
|
|
|
|
shape_id_t rb_shape_id(rb_shape_t * shape);
|
|
|
|
MJIT_SYMBOL_EXPORT_END
|
|
|
|
|
2022-11-08 23:35:31 +03:00
|
|
|
rb_shape_t * rb_shape_rebuild_shape(rb_shape_t * initial_shape, rb_shape_t * dest_shape);
|
|
|
|
|
|
|
|
static inline uint32_t
|
|
|
|
ROBJECT_IV_CAPACITY(VALUE obj)
|
|
|
|
{
|
|
|
|
RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
|
|
|
|
return rb_shape_get_shape_by_id(ROBJECT_SHAPE_ID(obj))->capacity;
|
|
|
|
}
|
|
|
|
|
2022-10-15 19:37:44 +03:00
|
|
|
static inline uint32_t
|
|
|
|
ROBJECT_IV_COUNT(VALUE obj)
|
|
|
|
{
|
|
|
|
RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
|
2022-10-21 23:24:29 +03:00
|
|
|
uint32_t ivc = rb_shape_get_shape_by_id(ROBJECT_SHAPE_ID(obj))->next_iv_index;
|
2022-10-15 19:37:44 +03:00
|
|
|
return ivc;
|
|
|
|
}
|
|
|
|
|
2022-11-08 23:35:31 +03:00
|
|
|
static inline uint32_t
|
|
|
|
RBASIC_IV_COUNT(VALUE obj)
|
|
|
|
{
|
|
|
|
return rb_shape_get_shape_by_id(rb_shape_get_shape_id(obj))->next_iv_index;
|
|
|
|
}
|
|
|
|
|
2022-11-01 00:05:37 +03:00
|
|
|
static inline uint32_t
|
|
|
|
RCLASS_IV_COUNT(VALUE obj)
|
|
|
|
{
|
|
|
|
RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE));
|
|
|
|
uint32_t ivc = rb_shape_get_shape_by_id(RCLASS_SHAPE_ID(obj))->next_iv_index;
|
|
|
|
return ivc;
|
|
|
|
}
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
rb_shape_t * rb_shape_alloc(ID edge_name, rb_shape_t * parent);
|
2022-11-08 23:35:31 +03:00
|
|
|
rb_shape_t * rb_shape_alloc_with_size_pool_index(ID edge_name, rb_shape_t * parent, uint8_t size_pool_index);
|
2022-10-03 20:52:40 +03:00
|
|
|
rb_shape_t * rb_shape_alloc_with_parent_id(ID edge_name, shape_id_t parent_id);
|
2022-10-03 18:14:32 +03:00
|
|
|
|
|
|
|
bool rb_shape_set_shape_id(VALUE obj, shape_id_t shape_id);
|
|
|
|
|
|
|
|
VALUE rb_obj_debug_shape(VALUE self, VALUE obj);
|
|
|
|
VALUE rb_shape_flags_mask(void);
|
|
|
|
|
|
|
|
#endif
|