From 71babe5536bdb2238509752d8706194ee57ff485 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 4 Dec 2023 09:26:26 -0500 Subject: [PATCH] Use 32-bit integers for redblack_id_t On 32-bit systems, the shape cache size is 1048576 (value of REDBLACK_CACHE_SIZE), but a 16-bit unsigned integer can only go up to 65536. This means that the redblack_id_t can overflow and lead to a corrupted red-black tree. The following script crashes on 32-bit systems: o = Object.new 1_000_000.times do |i| o.instance_variable_set(:"@i#{i}", i) end --- shape.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shape.h b/shape.h index a222a75ad8..4fc46b4525 100644 --- a/shape.h +++ b/shape.h @@ -9,7 +9,6 @@ #define SHAPE_IN_BASIC_FLAGS 1 typedef uint32_t attr_index_t; typedef uint32_t shape_id_t; -typedef uint32_t redblack_id_t; # define SHAPE_ID_NUM_BITS 32 #else @@ -18,11 +17,12 @@ typedef uint32_t redblack_id_t; #define SHAPE_IN_BASIC_FLAGS 0 typedef uint16_t attr_index_t; typedef uint16_t shape_id_t; -typedef uint16_t redblack_id_t; # define SHAPE_ID_NUM_BITS 16 #endif +typedef uint32_t redblack_id_t; + #define MAX_IVARS (attr_index_t)(-1) # define SHAPE_MASK (((uintptr_t)1 << SHAPE_ID_NUM_BITS) - 1)