Fix "too complex" iv sets on generic ivar objects

We weren't taking in to account that objects with generic IV tables
could go "too complex" in the IV set code.  This commit takes that in to
account and also ensures FL_EXIVAR is set when a geniv object
transitions to "too complex"

Co-Authored-By: Jean Boussier <byroot@ruby-lang.org>

2 files changed, 32 insertions, 0 deletions

diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb
index b9ac1cafe1..a098edf409 100644
--- a/test/ruby/test_shapes.rb
+++ b/test/ruby/test_shapes.rb
@@ -184,6 +184,32 @@ class TestShapes < Test::Unit::TestCase
     assert_empty obj.instance_variables
   end
 
+  def test_too_complex_geniv
+    assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
+    begin;
+      class TooComplex < Hash
+        attr_reader :very_unique
+      end
+
+      obj = Object.new
+      i = 0
+      while RubyVM::Shape.shapes_available > 0
+        obj.instance_variable_set(:"@a#{i}", 1)
+        i += 1
+      end
+
+      (RubyVM::Shape::SHAPE_MAX_VARIATIONS * 2).times do
+        TooComplex.new.instance_variable_set(:"@unique_#{_1}", 1)
+      end
+
+      tc = TooComplex.new
+      tc.instance_variable_set(:@very_unique, 3)
+      tc.instance_variable_set(:@very_unique2, 4)
+      assert_equal 3, tc.instance_variable_get(:@very_unique)
+      assert_equal 4, tc.instance_variable_get(:@very_unique2)
+    end;
+  end
+
   def test_use_all_shapes_then_freeze
     assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
     begin;
diff --git a/variable.c b/variable.c
index 34934a7b84..aa1b61ee50 100644
--- a/variable.c
+++ b/variable.c
@@ -1483,6 +1483,11 @@ generic_ivar_set(VALUE obj, ID id, VALUE val)
 
     attr_index_t index;
     // The returned shape will have `id` in its iv_table
+    if (rb_shape_obj_too_complex(obj)) {
+        rb_complex_ivar_set(obj, id, val);
+        return;
+    }
+
     rb_shape_t *shape = rb_shape_get_shape(obj);
     if (UNLIKELY(shape->type == SHAPE_OBJ_TOO_COMPLEX)) {
         rb_complex_ivar_set(obj, id, val);
@@ -1498,6 +1503,7 @@ generic_ivar_set(VALUE obj, ID id, VALUE val)
             rb_evict_ivars_to_hash(obj, shape);
             rb_complex_ivar_set(obj, id, val);
             rb_shape_set_shape(obj, next_shape);
+            FL_SET_RAW(obj, FL_EXIVAR);
             return;
         }
         shape = next_shape;