remove_instance_variable: Handle running out of shapes

`remove_shape_recursive` wasn't considering that if we run out of
shapes, it might have to transition to SHAPE_TOO_COMPLEX.

When this happens, we now return with an error and the caller
initiates the evacuation.

1 files changed, 13 insertions, 12 deletions

diff --git a/variable.c b/variable.c
index 41c2b3650c..be04b34527 100644
--- a/variable.c
+++ b/variable.c
@@ -2198,28 +2198,32 @@ rb_obj_remove_instance_variable(VALUE obj, VALUE name)
       case T_CLASS:
       case T_MODULE:
         IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id);
-        if (rb_shape_obj_too_complex(obj)) {
+        if (!rb_shape_transition_shape_remove_ivar(obj, id, shape, &val)) {
+            if (!rb_shape_obj_too_complex(obj)) {
+                rb_evict_ivars_to_hash(obj, shape);
+            }
             if (!st_delete(RCLASS_IV_HASH(obj), (st_data_t *)&id, (st_data_t *)&val)) {
                 val = Qundef;
             }
         }
-        else {
-            rb_shape_transition_shape_remove_ivar(obj, id, shape, &val);
-        }
         break;
       case T_OBJECT: {
-        if (rb_shape_obj_too_complex(obj)) {
+        if (!rb_shape_transition_shape_remove_ivar(obj, id, shape, &val)) {
+            if (!rb_shape_obj_too_complex(obj)) {
+                rb_evict_ivars_to_hash(obj, shape);
+            }
             if (rb_st_lookup(ROBJECT_IV_HASH(obj), (st_data_t)id, (st_data_t *)&val)) {
                 rb_st_delete(ROBJECT_IV_HASH(obj), (st_data_t *)&id, 0);
             }
         }
-        else {
-            rb_shape_transition_shape_remove_ivar(obj, id, shape, &val);
-        }
         break;
       }
       default: {
-        if (rb_shape_obj_too_complex(obj)) {
+        if (!rb_shape_transition_shape_remove_ivar(obj, id, shape, &val)) {
+            if (!rb_shape_obj_too_complex(obj)) {
+                rb_evict_ivars_to_hash(obj, shape);
+            }
+
             struct gen_ivtbl *ivtbl;
             if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) {
                 if (!st_delete(ivtbl->as.complex.table, (st_data_t *)&id, (st_data_t *)&val)) {
@@ -2227,9 +2231,6 @@ rb_obj_remove_instance_variable(VALUE obj, VALUE name)
                 }
             }
         }
-        else {
-            rb_shape_transition_shape_remove_ivar(obj, id, shape, &val);
-        }
         break;
       }
     }