Avoid improper optimization of case statements mixed integer/rational/complex

Fixes [Bug #17857]

compile.c

@@ -4537,6 +4537,9 @@ rb_node_case_when_optimizable_literal(const NODE *const node)
 	    modf(RFLOAT_VALUE(v), &ival) == 0.0) {
 	    return FIXABLE(ival) ? LONG2FIX((long)ival) : rb_dbl2big(ival);
 	}
+        if (RB_TYPE_P(v, T_RATIONAL) || RB_TYPE_P(v, T_COMPLEX)) {
+            return Qundef;
+        }
 	if (SYMBOL_P(v) || rb_obj_is_kind_of(v, rb_cNumeric)) {
 	    return v;
 	}

test/ruby/test_case.rb

@@ -102,6 +102,18 @@ class TestCase < Test::Unit::TestCase
     else
       assert(false)
     end
+    case 0
+    when 0r
+      assert(true)
+    else
+      assert(false)
+    end
+    case 0
+    when 0i
+      assert(true)
+    else
+      assert(false)
+    end
   end
 
   def test_method_missing