2018-10-31 00:53:56 +03:00
|
|
|
/**********************************************************************
|
|
|
|
|
|
|
|
transient_heap.c - implement transient_heap.
|
|
|
|
|
|
|
|
Copyright (C) 2018 Koichi Sasada
|
|
|
|
|
|
|
|
**********************************************************************/
|
|
|
|
|
2019-12-04 11:16:30 +03:00
|
|
|
#include "debug_counter.h"
|
2018-10-31 00:53:56 +03:00
|
|
|
#include "gc.h"
|
|
|
|
#include "internal.h"
|
2022-07-20 22:30:19 +03:00
|
|
|
#include "internal/array.h"
|
2019-12-04 11:16:30 +03:00
|
|
|
#include "internal/gc.h"
|
|
|
|
#include "internal/hash.h"
|
|
|
|
#include "internal/sanitizers.h"
|
|
|
|
#include "internal/static_assert.h"
|
|
|
|
#include "internal/struct.h"
|
|
|
|
#include "internal/variable.h"
|
|
|
|
#include "ruby/debug.h"
|
|
|
|
#include "ruby/ruby.h"
|
2018-10-31 00:53:56 +03:00
|
|
|
#include "ruby_assert.h"
|
|
|
|
#include "transient_heap.h"
|
2019-12-04 11:16:30 +03:00
|
|
|
#include "vm_debug.h"
|
2020-03-09 20:22:11 +03:00
|
|
|
#include "vm_sync.h"
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2018-11-01 11:53:44 +03:00
|
|
|
#if USE_TRANSIENT_HEAP /* USE_TRANSIENT_HEAP */
|
2018-10-31 00:53:56 +03:00
|
|
|
/*
|
|
|
|
* 1: enable assertions
|
|
|
|
* 2: enable verify all transient heaps
|
|
|
|
*/
|
|
|
|
#ifndef TRANSIENT_HEAP_CHECK_MODE
|
|
|
|
#define TRANSIENT_HEAP_CHECK_MODE 0
|
|
|
|
#endif
|
|
|
|
#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* 1: show events
|
|
|
|
* 2: show dump at events
|
|
|
|
* 3: show all operations
|
|
|
|
*/
|
|
|
|
#define TRANSIENT_HEAP_DEBUG 0
|
|
|
|
|
|
|
|
/* For Debug: Provide blocks infinitely.
|
|
|
|
* This mode generates blocks unlimitedly
|
|
|
|
* and prohibit access free'ed blocks to check invalid access.
|
|
|
|
*/
|
|
|
|
#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
|
|
|
|
|
|
|
|
#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
|
|
|
|
#include <sys/mman.h>
|
|
|
|
#include <errno.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* For Debug: Prohibit promoting to malloc space.
|
|
|
|
*/
|
|
|
|
#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
|
|
|
|
|
|
|
|
/* size configuration */
|
|
|
|
#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
|
|
|
|
|
|
|
|
/* K M */
|
|
|
|
#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */
|
2021-12-06 05:26:05 +03:00
|
|
|
#ifndef TRANSIENT_HEAP_TOTAL_SIZE
|
2018-10-31 00:53:56 +03:00
|
|
|
#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */
|
2021-12-06 05:26:05 +03:00
|
|
|
#endif
|
2018-10-31 00:53:56 +03:00
|
|
|
#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */
|
|
|
|
#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
|
2020-11-04 12:45:48 +03:00
|
|
|
#define TRANSIENT_HEAP_USABLE_SIZE (TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header))
|
2018-10-31 00:53:56 +03:00
|
|
|
|
|
|
|
#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
|
|
|
|
#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
|
|
|
|
|
|
|
|
#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
|
|
|
|
#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
|
|
|
|
|
|
|
|
enum transient_heap_status {
|
|
|
|
transient_heap_none,
|
|
|
|
transient_heap_marking,
|
|
|
|
transient_heap_escaping
|
|
|
|
};
|
|
|
|
|
|
|
|
struct transient_heap_block {
|
|
|
|
struct transient_heap_block_header {
|
|
|
|
int16_t index;
|
|
|
|
int16_t last_marked_index;
|
|
|
|
int16_t objects;
|
|
|
|
struct transient_heap_block *next_block;
|
|
|
|
} info;
|
2020-10-06 10:31:25 +03:00
|
|
|
char buff[TRANSIENT_HEAP_USABLE_SIZE];
|
2018-10-31 00:53:56 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
struct transient_heap {
|
|
|
|
struct transient_heap_block *using_blocks;
|
|
|
|
struct transient_heap_block *marked_blocks;
|
|
|
|
struct transient_heap_block *free_blocks;
|
|
|
|
int total_objects;
|
|
|
|
int total_marked_objects;
|
|
|
|
int total_blocks;
|
|
|
|
enum transient_heap_status status;
|
|
|
|
|
|
|
|
VALUE *promoted_objects;
|
|
|
|
int promoted_objects_size;
|
|
|
|
int promoted_objects_index;
|
|
|
|
|
|
|
|
struct transient_heap_block *arena;
|
|
|
|
int arena_index; /* increment only */
|
|
|
|
};
|
|
|
|
|
|
|
|
struct transient_alloc_header {
|
|
|
|
uint16_t magic;
|
|
|
|
uint16_t size;
|
|
|
|
int16_t next_marked_index;
|
|
|
|
int16_t dummy;
|
|
|
|
VALUE obj;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct transient_heap global_transient_heap;
|
|
|
|
|
|
|
|
static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj);
|
|
|
|
static const void *transient_heap_ptr(VALUE obj, int error);
|
|
|
|
static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr);
|
|
|
|
|
|
|
|
#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)
|
|
|
|
{
|
|
|
|
int i=0, n=0;
|
|
|
|
|
|
|
|
while (i<block->info.index) {
|
2018-10-31 06:19:37 +03:00
|
|
|
void *ptr = &block->buff[i];
|
|
|
|
struct transient_alloc_header *header = ptr;
|
|
|
|
fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, ptr, header->size, header->next_marked_index, rb_obj_info(header->obj));
|
2018-10-31 00:53:56 +03:00
|
|
|
i += header->size;
|
|
|
|
n++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
|
|
|
|
{
|
|
|
|
while (block) {
|
|
|
|
fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
|
2018-10-31 06:19:37 +03:00
|
|
|
type_str, (void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (void *)block->info.next_block);
|
2018-10-31 00:53:56 +03:00
|
|
|
|
|
|
|
transient_heap_block_dump(theap, block);
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_dump(struct transient_heap* theap)
|
|
|
|
{
|
|
|
|
fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
|
|
|
|
transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");
|
|
|
|
transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");
|
|
|
|
transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");
|
|
|
|
}
|
|
|
|
|
2020-11-18 21:04:09 +03:00
|
|
|
/* Debug: dump all transient_heap blocks */
|
2018-10-31 00:53:56 +03:00
|
|
|
void
|
|
|
|
rb_transient_heap_dump(void)
|
|
|
|
{
|
|
|
|
transient_heap_dump(&global_transient_heap);
|
|
|
|
}
|
|
|
|
|
|
|
|
#if TRANSIENT_HEAP_CHECK_MODE >= 2
|
2019-04-24 05:23:13 +03:00
|
|
|
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void transient_heap_ptr_check(struct transient_heap *theap, VALUE obj));
|
2018-10-31 00:53:56 +03:00
|
|
|
static void
|
|
|
|
transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)
|
|
|
|
{
|
2022-11-15 07:24:08 +03:00
|
|
|
if (!UNDEF_P(obj)) {
|
2018-10-31 00:53:56 +03:00
|
|
|
const void *ptr = transient_heap_ptr(obj, FALSE);
|
|
|
|
TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-24 05:23:13 +03:00
|
|
|
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static int transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block));
|
2018-10-31 00:53:56 +03:00
|
|
|
static int
|
|
|
|
transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)
|
|
|
|
{
|
|
|
|
int i=0, n=0;
|
|
|
|
struct transient_alloc_header *header;
|
|
|
|
|
|
|
|
while (i<block->info.index) {
|
|
|
|
header = (void *)&block->buff[i];
|
|
|
|
TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
|
|
|
|
transient_heap_ptr_check(theap, header->obj);
|
|
|
|
n ++;
|
|
|
|
i += header->size;
|
|
|
|
}
|
|
|
|
TH_ASSERT(block->info.objects == n);
|
|
|
|
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr)
|
|
|
|
{
|
|
|
|
int n = 0;
|
|
|
|
struct transient_heap_block *block = blocks;
|
|
|
|
while (block) {
|
|
|
|
n += transient_heap_block_verify(theap, block);
|
|
|
|
*block_num_ptr += 1;
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
|
|
|
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_verify(struct transient_heap *theap)
|
|
|
|
{
|
|
|
|
#if TRANSIENT_HEAP_CHECK_MODE >= 2
|
|
|
|
int n=0, block_num=0;
|
|
|
|
|
|
|
|
n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
|
|
|
|
n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
|
|
|
|
|
|
|
|
TH_ASSERT(n == theap->total_objects);
|
|
|
|
TH_ASSERT(n >= theap->total_marked_objects);
|
|
|
|
TH_ASSERT(block_num == theap->total_blocks);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Debug: check assertions for all transient_heap blocks */
|
|
|
|
void
|
|
|
|
rb_transient_heap_verify(void)
|
|
|
|
{
|
|
|
|
transient_heap_verify(&global_transient_heap);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_heap*
|
|
|
|
transient_heap_get(void)
|
|
|
|
{
|
|
|
|
struct transient_heap* theap = &global_transient_heap;
|
|
|
|
transient_heap_verify(theap);
|
|
|
|
return theap;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
reset_block(struct transient_heap_block *block)
|
|
|
|
{
|
2018-11-06 13:06:07 +03:00
|
|
|
__msan_allocated_memory(block, sizeof block);
|
2018-10-31 00:53:56 +03:00
|
|
|
block->info.index = 0;
|
|
|
|
block->info.objects = 0;
|
|
|
|
block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
|
|
|
|
block->info.next_block = NULL;
|
2018-11-06 13:06:07 +03:00
|
|
|
__asan_poison_memory_region(&block->buff, sizeof block->buff);
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)
|
|
|
|
{
|
|
|
|
block->info.next_block = theap->free_blocks;
|
|
|
|
theap->free_blocks = block;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)
|
|
|
|
{
|
|
|
|
block->info.next_block = theap->using_blocks;
|
|
|
|
theap->using_blocks = block;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if 0
|
|
|
|
static void
|
|
|
|
connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)
|
|
|
|
{
|
|
|
|
block->info.next_block = theap->marked_blocks;
|
|
|
|
theap->marked_blocks = block;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static void
|
|
|
|
append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)
|
|
|
|
{
|
|
|
|
if (theap->marked_blocks) {
|
|
|
|
struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;
|
|
|
|
while (block) {
|
|
|
|
last_block = block;
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
|
|
|
|
|
|
|
TH_ASSERT(last_block->info.next_block == NULL);
|
|
|
|
last_block->info.next_block = append_blocks;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
theap->marked_blocks = append_blocks;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_heap_block *
|
|
|
|
transient_heap_block_alloc(struct transient_heap* theap)
|
|
|
|
{
|
|
|
|
struct transient_heap_block *block;
|
|
|
|
#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
|
|
|
|
block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
|
|
|
|
MAP_PRIVATE | MAP_ANONYMOUS,
|
|
|
|
-1, 0);
|
|
|
|
if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);
|
|
|
|
#else
|
|
|
|
if (theap->arena == NULL) {
|
|
|
|
theap->arena = rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE);
|
2019-11-01 08:33:44 +03:00
|
|
|
if (theap->arena == NULL) {
|
|
|
|
rb_bug("transient_heap_block_alloc: failed\n");
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM);
|
|
|
|
block = &theap->arena[theap->arena_index++];
|
|
|
|
TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
|
|
|
|
#endif
|
|
|
|
reset_block(block);
|
|
|
|
|
|
|
|
TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);
|
2018-10-31 06:19:37 +03:00
|
|
|
if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, (void *)block);
|
2018-10-31 00:53:56 +03:00
|
|
|
return block;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static struct transient_heap_block *
|
|
|
|
transient_heap_allocatable_block(struct transient_heap* theap)
|
|
|
|
{
|
|
|
|
struct transient_heap_block *block;
|
|
|
|
|
|
|
|
#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
|
|
|
|
block = transient_heap_block_alloc(theap);
|
|
|
|
theap->total_blocks++;
|
|
|
|
#else
|
|
|
|
/* get one block from free_blocks */
|
|
|
|
block = theap->free_blocks;
|
|
|
|
if (block) {
|
|
|
|
theap->free_blocks = block->info.next_block;
|
|
|
|
block->info.next_block = NULL;
|
|
|
|
theap->total_blocks++;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return block;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_alloc_header *
|
|
|
|
transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
|
|
|
|
{
|
|
|
|
struct transient_heap_block *block = theap->using_blocks;
|
|
|
|
|
|
|
|
while (block) {
|
2020-11-04 12:45:48 +03:00
|
|
|
TH_ASSERT(block->info.index <= (int16_t)TRANSIENT_HEAP_USABLE_SIZE);
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-10-06 10:31:25 +03:00
|
|
|
if (TRANSIENT_HEAP_USABLE_SIZE - block->info.index >= size) {
|
2018-10-31 00:53:56 +03:00
|
|
|
struct transient_alloc_header *header = (void *)&block->buff[block->info.index];
|
|
|
|
block->info.index += size;
|
|
|
|
block->info.objects++;
|
|
|
|
return header;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
block = transient_heap_allocatable_block(theap);
|
|
|
|
if (block) connect_to_using_blocks(theap, block);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
void *
|
|
|
|
rb_transient_heap_alloc(VALUE obj, size_t req_size)
|
|
|
|
{
|
2020-12-05 00:15:17 +03:00
|
|
|
// only on single main ractor
|
|
|
|
if (ruby_single_main_ractor == NULL) return NULL;
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
void *ret;
|
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
TH_ASSERT(RB_TYPE_P(obj, T_ARRAY) ||
|
|
|
|
RB_TYPE_P(obj, T_OBJECT) ||
|
|
|
|
RB_TYPE_P(obj, T_STRUCT) ||
|
|
|
|
RB_TYPE_P(obj, T_HASH)); /* supported types */
|
2020-03-09 20:22:11 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
if (size > TRANSIENT_HEAP_ALLOC_MAX) {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj));
|
|
|
|
ret = NULL;
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
|
2020-12-05 00:15:17 +03:00
|
|
|
else if (RB_OBJ_PROMOTED_RAW(obj)) {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj));
|
|
|
|
ret = NULL;
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
#else
|
2020-12-05 00:15:17 +03:00
|
|
|
else if (RBASIC_CLASS(obj) == 0) {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj));
|
|
|
|
ret = NULL;
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
#endif
|
2020-12-05 00:15:17 +03:00
|
|
|
else {
|
|
|
|
struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);
|
|
|
|
if (header) {
|
|
|
|
void *ptr;
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
/* header is poisoned to prevent buffer overflow, should
|
|
|
|
* unpoison first... */
|
|
|
|
asan_unpoison_memory_region(header, sizeof *header, true);
|
2018-11-06 13:06:07 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
header->size = size;
|
|
|
|
header->magic = TRANSIENT_HEAP_ALLOC_MAGIC;
|
|
|
|
header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
|
|
|
|
header->obj = obj; /* TODO: can we eliminate it? */
|
2018-11-06 13:06:07 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
/* header is fixed; shall poison again */
|
|
|
|
asan_poison_memory_region(header, sizeof *header);
|
|
|
|
ptr = header + 1;
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
theap->total_objects++; /* statistics */
|
2018-10-31 00:53:56 +03:00
|
|
|
|
|
|
|
#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
|
2020-12-05 00:15:17 +03:00
|
|
|
if (RB_OBJ_PROMOTED_RAW(obj)) {
|
|
|
|
transient_heap_promote_add(theap, obj);
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
#endif
|
2020-12-05 00:15:17 +03:00
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", (void *)header, ptr, (int)size, rb_obj_info(obj));
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
RB_DEBUG_COUNTER_INC(theap_alloc);
|
2018-11-06 13:06:07 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
/* ptr is set up; OK to unpoison. */
|
|
|
|
asan_unpoison_memory_region(ptr, size - sizeof *header, true);
|
|
|
|
ret = ptr;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj));
|
|
|
|
RB_DEBUG_COUNTER_INC(theap_alloc_fail);
|
|
|
|
ret = NULL;
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
|
|
|
}
|
2020-03-09 20:22:11 +03:00
|
|
|
|
|
|
|
return ret;
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
Init_TransientHeap(void)
|
|
|
|
{
|
|
|
|
int i, block_num;
|
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
|
|
|
|
#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
|
|
|
|
block_num = 0;
|
|
|
|
#else
|
|
|
|
TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE);
|
|
|
|
block_num = TRANSIENT_HEAP_BLOCK_NUM;
|
|
|
|
#endif
|
|
|
|
for (i=0; i<block_num; i++) {
|
|
|
|
connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
|
|
|
|
}
|
|
|
|
theap->using_blocks = transient_heap_allocatable_block(theap);
|
|
|
|
|
|
|
|
theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE;
|
|
|
|
theap->promoted_objects_index = 0;
|
|
|
|
/* should not use ALLOC_N to be free from GC */
|
|
|
|
theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);
|
2019-10-07 10:56:08 +03:00
|
|
|
STATIC_ASSERT(
|
|
|
|
integer_overflow,
|
|
|
|
sizeof(VALUE) <= SIZE_MAX / TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE);
|
2018-10-31 00:53:56 +03:00
|
|
|
if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_heap_block *
|
|
|
|
blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)
|
|
|
|
{
|
|
|
|
struct transient_heap_block *block = blocks;
|
|
|
|
|
|
|
|
while (block) {
|
2020-10-06 10:31:25 +03:00
|
|
|
if (block->buff <= (char *)header && (char *)header < block->buff + TRANSIENT_HEAP_USABLE_SIZE) {
|
2018-10-31 00:53:56 +03:00
|
|
|
return block;
|
|
|
|
}
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_heap_block *
|
|
|
|
alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header)
|
|
|
|
{
|
|
|
|
struct transient_heap_block *block;
|
|
|
|
|
|
|
|
if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");
|
|
|
|
return block;
|
|
|
|
}
|
|
|
|
else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");
|
|
|
|
return block;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_alloc_header *
|
|
|
|
ptr_to_alloc_header(const void *ptr)
|
|
|
|
{
|
|
|
|
struct transient_alloc_header *header = (void *)ptr;
|
|
|
|
header -= 1;
|
|
|
|
return header;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr)
|
|
|
|
{
|
|
|
|
if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) {
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int
|
|
|
|
rb_transient_heap_managed_ptr_p(const void *ptr)
|
|
|
|
{
|
|
|
|
return transient_header_managed_ptr_p(transient_heap_get(), ptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_heap_block *
|
|
|
|
alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)
|
|
|
|
{
|
|
|
|
struct transient_heap_block *block;
|
|
|
|
#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
|
|
|
|
block = alloc_header_to_block_verbose(theap, header);
|
|
|
|
if (block == NULL) {
|
|
|
|
transient_heap_dump(theap);
|
|
|
|
rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
|
|
|
|
TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
|
|
|
|
#endif
|
|
|
|
return block;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rb_transient_heap_mark(VALUE obj, const void *ptr)
|
|
|
|
{
|
2020-03-09 20:22:11 +03:00
|
|
|
ASSERT_vm_locking();
|
|
|
|
|
2018-10-31 00:53:56 +03:00
|
|
|
struct transient_alloc_header *header = ptr_to_alloc_header(ptr);
|
2019-05-23 11:02:07 +03:00
|
|
|
asan_unpoison_memory_region(header, sizeof *header, false);
|
2018-10-31 01:11:51 +03:00
|
|
|
if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header, %s (%p)", rb_obj_info(obj), ptr);
|
2018-10-31 00:53:56 +03:00
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);
|
|
|
|
|
|
|
|
#if TRANSIENT_HEAP_CHECK_MODE > 0
|
|
|
|
{
|
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
TH_ASSERT(theap->status == transient_heap_marking);
|
|
|
|
TH_ASSERT(transient_header_managed_ptr_p(theap, ptr));
|
|
|
|
|
|
|
|
if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
|
|
|
|
transient_heap_dump(theap);
|
|
|
|
rb_bug("rb_transient_heap_mark: magic is broken");
|
|
|
|
}
|
|
|
|
else if (header->obj != obj) {
|
2018-10-31 01:11:51 +03:00
|
|
|
// transient_heap_dump(theap);
|
2018-10-31 00:53:56 +03:00
|
|
|
rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
|
|
|
|
rb_obj_info(header->obj), rb_obj_info(obj));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) {
|
|
|
|
/* already marked */
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
struct transient_heap_block *block = alloc_header_to_block(theap, header);
|
2018-11-06 13:06:07 +03:00
|
|
|
__asan_unpoison_memory_region(&block->info, sizeof block->info);
|
2018-10-31 00:53:56 +03:00
|
|
|
header->next_marked_index = block->info.last_marked_index;
|
|
|
|
block->info.last_marked_index = (int)((char *)header - block->buff);
|
|
|
|
theap->total_marked_objects++;
|
|
|
|
|
|
|
|
transient_heap_verify(theap);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-24 05:23:13 +03:00
|
|
|
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error));
|
2018-10-31 00:53:56 +03:00
|
|
|
static const void *
|
|
|
|
transient_heap_ptr(VALUE obj, int error)
|
|
|
|
{
|
2018-10-31 01:03:42 +03:00
|
|
|
const void *ptr = NULL;
|
2018-10-31 00:53:56 +03:00
|
|
|
|
|
|
|
switch (BUILTIN_TYPE(obj)) {
|
|
|
|
case T_ARRAY:
|
|
|
|
if (RARRAY_TRANSIENT_P(obj)) {
|
2022-07-20 22:30:19 +03:00
|
|
|
TH_ASSERT(!ARY_EMBED_P(obj));
|
2018-10-31 00:53:56 +03:00
|
|
|
ptr = RARRAY(obj)->as.heap.ptr;
|
|
|
|
}
|
|
|
|
break;
|
2018-10-31 01:01:17 +03:00
|
|
|
case T_OBJECT:
|
|
|
|
if (ROBJ_TRANSIENT_P(obj)) {
|
2022-12-09 01:16:52 +03:00
|
|
|
RUBY_ASSERT(!rb_shape_obj_too_complex(obj));
|
2018-10-31 01:01:17 +03:00
|
|
|
ptr = ROBJECT_IVPTR(obj);
|
|
|
|
}
|
2018-10-31 01:03:42 +03:00
|
|
|
break;
|
|
|
|
case T_STRUCT:
|
|
|
|
if (RSTRUCT_TRANSIENT_P(obj)) {
|
|
|
|
ptr = rb_struct_const_heap_ptr(obj);
|
2018-10-31 01:01:17 +03:00
|
|
|
}
|
|
|
|
break;
|
2018-10-31 01:11:51 +03:00
|
|
|
case T_HASH:
|
|
|
|
if (RHASH_TRANSIENT_P(obj)) {
|
2018-12-14 04:10:15 +03:00
|
|
|
TH_ASSERT(RHASH_AR_TABLE_P(obj));
|
|
|
|
ptr = (VALUE *)(RHASH(obj)->as.ar);
|
2018-10-31 01:12:12 +03:00
|
|
|
}
|
|
|
|
else {
|
|
|
|
ptr = NULL;
|
|
|
|
}
|
|
|
|
break;
|
2018-10-31 00:53:56 +03:00
|
|
|
default:
|
|
|
|
if (error) {
|
|
|
|
rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_promote_add(struct transient_heap* theap, VALUE obj)
|
|
|
|
{
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));
|
|
|
|
|
|
|
|
if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) {
|
|
|
|
/* duplicate check */
|
|
|
|
int i;
|
|
|
|
for (i=0; i<theap->promoted_objects_index; i++) {
|
|
|
|
if (theap->promoted_objects[i] == obj) return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (theap->promoted_objects_size <= theap->promoted_objects_index) {
|
|
|
|
theap->promoted_objects_size *= 2;
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
|
2019-10-07 10:56:08 +03:00
|
|
|
if (UNLIKELY((size_t)theap->promoted_objects_size > SIZE_MAX / sizeof(VALUE))) {
|
|
|
|
/* realloc failure due to integer overflow */
|
|
|
|
theap->promoted_objects = NULL;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");
|
|
|
|
}
|
|
|
|
theap->promoted_objects[theap->promoted_objects_index++] = obj;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rb_transient_heap_promote(VALUE obj)
|
|
|
|
{
|
2020-03-09 20:22:11 +03:00
|
|
|
ASSERT_vm_locking();
|
|
|
|
|
2018-10-31 00:53:56 +03:00
|
|
|
if (transient_heap_ptr(obj, FALSE)) {
|
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
transient_heap_promote_add(theap, obj);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
/* ignore */
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct transient_alloc_header *
|
|
|
|
alloc_header(struct transient_heap_block* block, int index)
|
|
|
|
{
|
|
|
|
return (void *)&block->buff[index];
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_reset(void)
|
|
|
|
{
|
2020-03-09 20:22:11 +03:00
|
|
|
ASSERT_vm_locking();
|
|
|
|
|
2018-10-31 00:53:56 +03:00
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
struct transient_heap_block* block;
|
|
|
|
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");
|
|
|
|
|
|
|
|
block = theap->marked_blocks;
|
|
|
|
while (block) {
|
|
|
|
struct transient_heap_block *next_block = block->info.next_block;
|
|
|
|
theap->total_objects -= block->info.objects;
|
|
|
|
#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
|
|
|
|
if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
|
|
|
|
rb_bug("madvise err:%d", errno);
|
|
|
|
}
|
|
|
|
if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
|
|
|
|
rb_bug("mprotect err:%d", errno);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
reset_block(block);
|
|
|
|
connect_to_free_blocks(theap, block);
|
|
|
|
#endif
|
|
|
|
theap->total_blocks--;
|
|
|
|
block = next_block;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks);
|
|
|
|
|
|
|
|
theap->marked_blocks = NULL;
|
|
|
|
theap->total_marked_objects = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block)
|
|
|
|
{
|
|
|
|
int marked_index = block->info.last_marked_index;
|
|
|
|
block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
|
|
|
|
|
|
|
|
while (marked_index >= 0) {
|
|
|
|
struct transient_alloc_header *header = alloc_header(block, marked_index);
|
2020-09-28 19:53:30 +03:00
|
|
|
asan_unpoison_memory_region(header, sizeof *header, true);
|
2018-10-31 00:53:56 +03:00
|
|
|
VALUE obj = header->obj;
|
2018-10-31 01:12:12 +03:00
|
|
|
TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
|
2020-11-18 21:04:09 +03:00
|
|
|
if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("transient_heap_block_evacuate: wrong header %p %s\n", (void *)header, rb_obj_info(obj));
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2018-10-31 06:19:37 +03:00
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", (void *)header, rb_obj_info(obj));
|
2018-10-31 00:53:56 +03:00
|
|
|
|
|
|
|
if (obj != Qnil) {
|
|
|
|
RB_DEBUG_COUNTER_INC(theap_evacuate);
|
|
|
|
|
|
|
|
switch (BUILTIN_TYPE(obj)) {
|
|
|
|
case T_ARRAY:
|
2018-10-31 01:01:17 +03:00
|
|
|
rb_ary_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
|
|
|
|
break;
|
|
|
|
case T_OBJECT:
|
|
|
|
rb_obj_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
|
2018-10-31 00:53:56 +03:00
|
|
|
break;
|
2018-10-31 01:03:42 +03:00
|
|
|
case T_STRUCT:
|
|
|
|
rb_struct_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
|
|
|
|
break;
|
2018-10-31 01:11:51 +03:00
|
|
|
case T_HASH:
|
|
|
|
rb_hash_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE);
|
|
|
|
break;
|
2018-10-31 00:53:56 +03:00
|
|
|
default:
|
2020-11-18 21:04:09 +03:00
|
|
|
rb_bug("unsupported: %s\n", rb_obj_info(obj));
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
|
|
|
header->obj = Qundef; /* for debug */
|
|
|
|
}
|
|
|
|
marked_index = header->next_marked_index;
|
2020-09-28 19:53:30 +03:00
|
|
|
asan_poison_memory_region(header, sizeof *header);
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-28 11:58:39 +03:00
|
|
|
#if USE_RUBY_DEBUG_LOG
|
2020-07-29 11:45:02 +03:00
|
|
|
static const char *
|
|
|
|
transient_heap_status_cstr(enum transient_heap_status status)
|
|
|
|
{
|
|
|
|
switch (status) {
|
|
|
|
case transient_heap_none: return "none";
|
|
|
|
case transient_heap_marking: return "marking";
|
|
|
|
case transient_heap_escaping: return "escaping";
|
|
|
|
}
|
2020-07-29 15:08:48 +03:00
|
|
|
UNREACHABLE_RETURN(NULL);
|
2020-07-29 11:45:02 +03:00
|
|
|
}
|
2020-07-29 15:08:48 +03:00
|
|
|
#endif
|
2020-07-29 11:45:02 +03:00
|
|
|
|
2018-10-31 00:53:56 +03:00
|
|
|
static void
|
|
|
|
transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)
|
|
|
|
{
|
2020-07-29 11:45:02 +03:00
|
|
|
RUBY_DEBUG_LOG("%s -> %s",
|
|
|
|
transient_heap_status_cstr(theap->status),
|
|
|
|
transient_heap_status_cstr(status));
|
|
|
|
|
2018-10-31 00:53:56 +03:00
|
|
|
TH_ASSERT(theap->status != status);
|
|
|
|
theap->status = status;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_evacuate(void *dmy)
|
|
|
|
{
|
2020-12-05 00:15:17 +03:00
|
|
|
struct transient_heap* theap = transient_heap_get();
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
if (theap->total_marked_objects == 0) return;
|
|
|
|
if (ruby_single_main_ractor == NULL) rb_bug("not single ractor mode");
|
|
|
|
if (theap->status == transient_heap_marking) {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
VALUE gc_disabled = rb_gc_disable_no_rest();
|
|
|
|
{
|
2020-03-09 20:22:11 +03:00
|
|
|
struct transient_heap_block* block;
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-03-09 20:22:11 +03:00
|
|
|
RUBY_DEBUG_LOG("start gc_disabled:%d", RTEST(gc_disabled));
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-03-09 20:22:11 +03:00
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 1) {
|
|
|
|
int i;
|
|
|
|
fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 4) {
|
|
|
|
for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i]));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
|
|
|
|
|
|
|
|
TH_ASSERT(theap->status == transient_heap_none);
|
|
|
|
transient_heap_update_status(theap, transient_heap_escaping);
|
|
|
|
|
|
|
|
/* evacuate from marked blocks */
|
|
|
|
block = theap->marked_blocks;
|
|
|
|
while (block) {
|
|
|
|
transient_heap_block_evacuate(theap, block);
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-03-09 20:22:11 +03:00
|
|
|
/* evacuate from using blocks
|
2018-10-31 00:53:56 +03:00
|
|
|
only affect incremental marking */
|
2020-03-09 20:22:11 +03:00
|
|
|
block = theap->using_blocks;
|
|
|
|
while (block) {
|
|
|
|
transient_heap_block_evacuate(theap, block);
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-03-09 20:22:11 +03:00
|
|
|
/* all objects in marked_objects are escaped. */
|
|
|
|
transient_heap_reset();
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-03-09 20:22:11 +03:00
|
|
|
if (TRANSIENT_HEAP_DEBUG > 0) {
|
|
|
|
fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
|
|
|
|
}
|
2018-10-31 00:53:56 +03:00
|
|
|
|
2020-03-09 20:22:11 +03:00
|
|
|
transient_heap_verify(theap);
|
|
|
|
transient_heap_update_status(theap, transient_heap_none);
|
|
|
|
}
|
2020-12-05 00:15:17 +03:00
|
|
|
if (gc_disabled != Qtrue) rb_gc_enable();
|
2021-09-28 12:00:03 +03:00
|
|
|
RUBY_DEBUG_LOG("finish");
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
2020-12-05 00:15:17 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rb_transient_heap_evacuate(void)
|
|
|
|
{
|
|
|
|
transient_heap_evacuate(NULL);
|
2018-10-31 00:53:56 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
clear_marked_index(struct transient_heap_block* block)
|
|
|
|
{
|
|
|
|
int marked_index = block->info.last_marked_index;
|
|
|
|
|
|
|
|
while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
|
|
|
|
struct transient_alloc_header *header = alloc_header(block, marked_index);
|
2019-04-24 10:47:09 +03:00
|
|
|
/* header is poisoned to prevent buffer overflow, should
|
|
|
|
* unpoison first... */
|
2019-05-23 11:02:07 +03:00
|
|
|
asan_unpoison_memory_region(header, sizeof *header, false);
|
2018-10-31 00:53:56 +03:00
|
|
|
TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE);
|
2018-10-31 06:19:37 +03:00
|
|
|
if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", (void *)block, marked_index);
|
2018-10-31 00:53:56 +03:00
|
|
|
|
|
|
|
marked_index = header->next_marked_index;
|
|
|
|
header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
|
|
|
|
}
|
|
|
|
|
|
|
|
block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
blocks_clear_marked_index(struct transient_heap_block* block)
|
|
|
|
{
|
|
|
|
while (block) {
|
|
|
|
clear_marked_index(block);
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-20 04:19:47 +03:00
|
|
|
static void
|
|
|
|
transient_heap_block_update_refs(struct transient_heap* theap, struct transient_heap_block* block)
|
|
|
|
{
|
2020-11-03 01:40:29 +03:00
|
|
|
int marked_index = block->info.last_marked_index;
|
2019-04-20 04:19:47 +03:00
|
|
|
|
2020-11-03 01:40:29 +03:00
|
|
|
while (marked_index >= 0) {
|
|
|
|
struct transient_alloc_header *header = alloc_header(block, marked_index);
|
2019-04-20 04:19:47 +03:00
|
|
|
|
2019-05-23 11:02:07 +03:00
|
|
|
asan_unpoison_memory_region(header, sizeof *header, false);
|
2019-04-20 04:19:47 +03:00
|
|
|
|
2019-05-18 12:23:47 +03:00
|
|
|
header->obj = rb_gc_location(header->obj);
|
2019-04-20 04:19:47 +03:00
|
|
|
|
2020-11-03 01:40:29 +03:00
|
|
|
marked_index = header->next_marked_index;
|
2019-05-23 11:02:07 +03:00
|
|
|
asan_poison_memory_region(header, sizeof *header);
|
2019-04-20 04:19:47 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
transient_heap_blocks_update_refs(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
|
|
|
|
{
|
|
|
|
while (block) {
|
|
|
|
transient_heap_block_update_refs(theap, block);
|
|
|
|
block = block->info.next_block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rb_transient_heap_update_references(void)
|
|
|
|
{
|
2020-03-09 20:22:11 +03:00
|
|
|
ASSERT_vm_locking();
|
|
|
|
|
2019-04-20 04:19:47 +03:00
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
int i;
|
|
|
|
|
|
|
|
transient_heap_blocks_update_refs(theap, theap->using_blocks, "using_blocks");
|
|
|
|
transient_heap_blocks_update_refs(theap, theap->marked_blocks, "marked_blocks");
|
|
|
|
|
|
|
|
for (i=0; i<theap->promoted_objects_index; i++) {
|
|
|
|
VALUE obj = theap->promoted_objects[i];
|
2019-05-18 12:23:47 +03:00
|
|
|
theap->promoted_objects[i] = rb_gc_location(obj);
|
2019-04-20 04:19:47 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-31 00:53:56 +03:00
|
|
|
void
|
|
|
|
rb_transient_heap_start_marking(int full_marking)
|
|
|
|
{
|
2020-03-09 20:22:11 +03:00
|
|
|
ASSERT_vm_locking();
|
2020-07-29 11:45:02 +03:00
|
|
|
RUBY_DEBUG_LOG("full?:%d", full_marking);
|
|
|
|
|
2018-10-31 00:53:56 +03:00
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
|
2020-11-18 21:04:09 +03:00
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promoted:%d full_marking:%d\n",
|
2018-10-31 00:53:56 +03:00
|
|
|
theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
|
|
|
|
|
|
|
|
blocks_clear_marked_index(theap->marked_blocks);
|
|
|
|
blocks_clear_marked_index(theap->using_blocks);
|
|
|
|
|
|
|
|
if (theap->using_blocks) {
|
|
|
|
if (theap->using_blocks->info.objects > 0) {
|
|
|
|
append_to_marked_blocks(theap, theap->using_blocks);
|
|
|
|
theap->using_blocks = NULL;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
|
|
|
|
theap->using_blocks->info.next_block = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (theap->using_blocks == NULL) {
|
|
|
|
theap->using_blocks = transient_heap_allocatable_block(theap);
|
|
|
|
}
|
|
|
|
|
|
|
|
TH_ASSERT(theap->status == transient_heap_none);
|
|
|
|
transient_heap_update_status(theap, transient_heap_marking);
|
|
|
|
theap->total_marked_objects = 0;
|
|
|
|
|
|
|
|
if (full_marking) {
|
|
|
|
theap->promoted_objects_index = 0;
|
|
|
|
}
|
|
|
|
else { /* mark promoted objects */
|
|
|
|
int i;
|
|
|
|
for (i=0; i<theap->promoted_objects_index; i++) {
|
|
|
|
VALUE obj = theap->promoted_objects[i];
|
|
|
|
const void *ptr = transient_heap_ptr(obj, TRUE);
|
|
|
|
if (ptr) {
|
|
|
|
rb_transient_heap_mark(obj, ptr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
transient_heap_verify(theap);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rb_transient_heap_finish_marking(void)
|
|
|
|
{
|
2020-03-09 20:22:11 +03:00
|
|
|
ASSERT_vm_locking();
|
2018-10-31 00:53:56 +03:00
|
|
|
struct transient_heap* theap = transient_heap_get();
|
|
|
|
|
2020-12-05 00:15:17 +03:00
|
|
|
RUBY_DEBUG_LOG("objects:%d, marked:%d",
|
|
|
|
theap->total_objects,
|
|
|
|
theap->total_marked_objects);
|
2018-10-31 00:53:56 +03:00
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
|
|
|
|
|
|
|
|
TH_ASSERT(theap->total_objects >= theap->total_marked_objects);
|
|
|
|
|
|
|
|
TH_ASSERT(theap->status == transient_heap_marking);
|
|
|
|
transient_heap_update_status(theap, transient_heap_none);
|
|
|
|
|
|
|
|
if (theap->total_marked_objects > 0) {
|
|
|
|
if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");
|
|
|
|
rb_postponed_job_register_one(0, transient_heap_evacuate, NULL);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
transient_heap_reset();
|
|
|
|
}
|
|
|
|
|
|
|
|
transient_heap_verify(theap);
|
|
|
|
}
|
2018-11-01 11:53:44 +03:00
|
|
|
#endif /* USE_TRANSIENT_HEAP */
|