compar.c: fail if recursion

* compar.c (cmp_eq): fail if recursion.  [ruby-core:57736] [Bug #9003]
* thread.c (rb_exec_recursive_paired_outer): new function which is
  combinnation of paired and outer variants.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@43208 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,10 @@
+Wed Oct  9 13:53:14 2013  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* compar.c (cmp_eq): fail if recursion.  [ruby-core:57736] [Bug #9003]
+
+	* thread.c (rb_exec_recursive_paired_outer): new function which is
+	  combinnation of paired and outer variants.
+
 Wed Oct  9 09:18:14 2013  Koichi Sasada  <ko1@atdot.net>
 
 	* include/ruby/debug.h,

compar.c

@@ -51,10 +51,17 @@ rb_invcmp(VALUE x, VALUE y)
     }
 }
 
+static VALUE
+cmp_eq_recursive(VALUE arg1, VALUE arg2, int recursive)
+{
+    if (recursive) return Qfalse;
+    return rb_funcallv(arg1, cmp, 1, &arg2);
+}
+
 static VALUE
 cmp_eq(VALUE *a)
 {
-    VALUE c = rb_funcall(a[0], cmp, 1, a[1]);
+    VALUE c = rb_exec_recursive_paired_outer(cmp_eq_recursive, a[0], a[1], a[1]);
 
     if (NIL_P(c)) return Qfalse;
     if (rb_cmpint(c, a[0], a[1]) == 0) return Qtrue;

include/ruby/intern.h

@@ -448,6 +448,7 @@ void rb_thread_atfork_before_exec(void);
 VALUE rb_exec_recursive(VALUE(*)(VALUE, VALUE, int),VALUE,VALUE);
 VALUE rb_exec_recursive_paired(VALUE(*)(VALUE, VALUE, int),VALUE,VALUE,VALUE);
 VALUE rb_exec_recursive_outer(VALUE(*)(VALUE, VALUE, int),VALUE,VALUE);
+VALUE rb_exec_recursive_paired_outer(VALUE(*)(VALUE, VALUE, int),VALUE,VALUE,VALUE);
 /* dir.c */
 VALUE rb_dir_getwd(void);
 /* file.c */

test/ruby/test_comparable.rb

@@ -76,4 +76,11 @@ class TestComparable < Test::Unit::TestCase
       assert_nil(Time.new <=> "")
     }
   end
+
+  def test_no_cmp
+    bug9003 = '[ruby-core:57736] [Bug #9003]'
+    assert_nothing_raised(SystemStackError, bug9003) {
+      @o <=> @o.dup
+    }
+  end
 end

thread.c

@@ -4953,6 +4953,18 @@ rb_exec_recursive_outer(VALUE (*func) (VALUE, VALUE, int), VALUE obj, VALUE arg)
     return exec_recursive(func, obj, 0, arg, 1);
 }
 
+/*
+ * If recursion is detected on the current method, obj and paired_obj,
+ * the outermost func will be called with (obj, arg, Qtrue). All inner
+ * func will be short-circuited using throw.
+ */
+
+VALUE
+rb_exec_recursive_paired_outer(VALUE (*func) (VALUE, VALUE, int), VALUE obj, VALUE paired_obj, VALUE arg)
+{
+    return exec_recursive(func, obj, rb_obj_id(paired_obj), arg, 1);
+}
+
 /*
  *  call-seq:
  *     thread.backtrace    -> array