Do not allow array modifications after freeze inside sort!

If freezing an array inside sort!, previously the array could be
modified after the freeze.  This checks whether the receiver is
frozen after every yield and potential call to #> or #<,
preventing modifications if the receiver is frozen inside the
block or by the #> or #< call.

Fixes [Bug #17739]

array.c

@@ -3213,6 +3213,7 @@ rb_ary_rotate_m(int argc, VALUE *argv, VALUE ary)
 
 struct ary_sort_data {
     VALUE ary;
+    VALUE receiver;
     struct cmp_opt_data cmp_opt;
 };
 
@@ -3225,6 +3226,15 @@ sort_reentered(VALUE ary)
     return Qnil;
 }
 
+static void
+sort_returned(struct ary_sort_data *data)
+{
+    if (rb_obj_frozen_p(data->receiver)) {
+        rb_raise(rb_eFrozenError, "array frozen during sort");
+    }
+    sort_reentered(data->ary);
+}
+
 static int
 sort_1(const void *ap, const void *bp, void *dummy)
 {
@@ -3238,7 +3248,7 @@ sort_1(const void *ap, const void *bp, void *dummy)
     args[1] = b;
     retval = rb_yield_values2(2, args);
     n = rb_cmpint(retval, a, b);
-    sort_reentered(data->ary);
+    sort_returned(data);
     return n;
 }
 
@@ -3264,7 +3274,7 @@ sort_2(const void *ap, const void *bp, void *dummy)
 
     retval = rb_funcallv(a, id_cmp, 1, &b);
     n = rb_cmpint(retval, a, b);
-    sort_reentered(data->ary);
+    sort_returned(data);
 
     return n;
 }
@@ -3316,6 +3326,7 @@ rb_ary_sort_bang(VALUE ary)
 	long len = RARRAY_LEN(ary);
 	RBASIC_CLEAR_CLASS(tmp);
 	data.ary = tmp;
+        data.receiver = ary;
 	data.cmp_opt.opt_methods = 0;
 	data.cmp_opt.opt_inited = 0;
 	RARRAY_PTR_USE(tmp, ptr, {

test/ruby/test_array.rb

@@ -1663,6 +1663,36 @@ class TestArray < Test::Unit::TestCase
     assert_equal([1, 2, 3, 4], a)
   end
 
+  def test_freeze_inside_sort!
+    array = [1, 2, 3, 4, 5]
+    frozen_array = nil
+    assert_raise(FrozenError) do
+      array.sort! do |a, b|
+        array.freeze if a == 3
+        frozen_array ||= array.map.to_a if array.frozen?
+        1
+      end
+    end
+    assert_equal(frozen_array, array)
+
+    object = Object.new
+    array = [1, 2, 3, 4, 5]
+    object.define_singleton_method(:>){|_| array.freeze; true}
+    assert_raise(FrozenError) do
+      array.sort! do |a, b|
+        object
+      end
+    end
+
+    object = Object.new
+    array = [object, object]
+    object.define_singleton_method(:>){|_| array.freeze; true}
+    object.define_singleton_method(:<=>){|o| object}
+    assert_raise(FrozenError) do
+      array.sort!
+    end
+  end
+
   def test_sort_with_callcc
     need_continuation
     n = 1000