hash.c: fix Hash#compact! return value

* hash.c (rb_hash_compact_bang): should return nil if no elements
  is deleted.  [ruby-core:77709] [Bug #12863]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56473 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,8 @@
+Sat Oct 22 18:52:32 2016  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* hash.c (rb_hash_compact_bang): should return nil if no elements
+	  is deleted.  [ruby-core:77709] [Bug #12863]
+
 Sat Oct 22 10:28:28 2016  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* configure.in (DLDFLAGS): fallback to LDFLAGS.

hash.c

@@ -2726,9 +2726,12 @@ rb_hash_compact_bang(VALUE hash)
 {
     rb_hash_modify_check(hash);
     if (RHASH(hash)->ntbl) {
+	st_index_t n = RHASH(hash)->ntbl->num_entries;
 	rb_hash_foreach(hash, delete_if_nil, hash);
+	if (n != RHASH(hash)->ntbl->num_entries)
+	    return hash;
     }
-    return hash;
+    return Qnil;
 }
 
 static VALUE rb_hash_compare_by_id_p(VALUE hash);

test/ruby/test_hash.rb

@@ -358,8 +358,9 @@ class TestHash < Test::Unit::TestCase
     h = @cls[a: 1, b: nil, c: false, d: true, e: nil]
     assert_equal({a: 1, c: false, d: true}, h.compact)
     assert_equal({a: 1, b: nil, c: false, d: true, e: nil}, h)
-    h.compact!
+    assert_same(h, h.compact!)
     assert_equal({a: 1, c: false, d: true}, h)
+    assert_nil(h.compact!)
   end
 
   def test_dup