generic_ivar_set: properly check for TOO_COMPLEX on capacity transition

test/ruby/test_shapes.rb

@@ -318,7 +318,7 @@ class TestShapes < Test::Unit::TestCase
     end;
   end
 
-  def test_run_out_of_shape_generic_ivar_set
+  def test_run_out_of_shape_generic_instance_variable_set
     assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
     begin;
       class TooComplex < Hash
@@ -345,6 +345,36 @@ class TestShapes < Test::Unit::TestCase
     end;
   end
 
+  def test_run_out_of_shape_generic_ivar_set
+    assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
+    begin;
+      class Hi < String
+        def initialize
+          8.times do |i|
+            instance_variable_set("@ivar_#{i}", i)
+          end
+        end
+
+        def transition
+          @hi_transition ||= 1
+        end
+      end
+
+      a = Hi.new
+
+      # Try to run out of shapes
+      o = Object.new
+      i = 0
+      while RubyVM::Shape.shapes_available > 0
+        o.instance_variable_set(:"@i#{i}", 1)
+        i += 1
+      end
+
+      assert_equal 1,a.transition
+      assert_equal 1,a.transition
+    end;
+  end
+
   def test_run_out_of_shape_instance_variable_defined
     assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
     begin;

variable.c

@@ -1516,7 +1516,7 @@ generic_ivar_set(VALUE obj, ID id, VALUE val)
             RUBY_ASSERT(index == shape->capacity);
 
             rb_shape_t *next_shape = rb_shape_transition_shape_capa(shape);
-            if (shape->type == SHAPE_OBJ_TOO_COMPLEX) {
+            if (next_shape->type == SHAPE_OBJ_TOO_COMPLEX) {
                 rb_evict_ivars_to_hash(obj, shape);
                 rb_complex_ivar_set(obj, id, val);
                 FL_SET_RAW(obj, FL_EXIVAR);