* Avoid undefined behaviors found by gcc -fsanitize=undefined.

gcc (Debian 4.9.1-16) 4.9.1

* include/ruby/ruby.h (INT2FIX): Avoid undefined behavior.

* node.h (nd_set_line): Ditto.

* pack.c (encodes): Ditto.
  (pack_unpack): Ditto.

* regint.h (BIT_STATUS_AT): Ditto.
  (BS_BIT): Ditto.

* time.c (time_mdump): Ditto.
  (time_mload): Ditto.

* vm_core.h (VM_FRAME_MAGIC_MASK): Ditto.

* vm_trace.c (recalc_add_ruby_vm_event_flags): Ditto.



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@47996 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,25 @@
+Fri Oct 17 17:43:50 2014  Tanaka Akira  <akr@fsij.org>
+
+	* Avoid undefined behaviors found by gcc -fsanitize=undefined.
+	  gcc (Debian 4.9.1-16) 4.9.1
+
+	* include/ruby/ruby.h (INT2FIX): Avoid undefined behavior.
+
+	* node.h (nd_set_line): Ditto.
+
+	* pack.c (encodes): Ditto.
+	  (pack_unpack): Ditto.
+
+	* regint.h (BIT_STATUS_AT): Ditto.
+	  (BS_BIT): Ditto.
+
+	* time.c (time_mdump): Ditto.
+	  (time_mload): Ditto.
+
+	* vm_core.h (VM_FRAME_MAGIC_MASK): Ditto.
+
+	* vm_trace.c (recalc_add_ruby_vm_event_flags): Ditto.
+
 Fri Oct 17 15:06:49 2014  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* re.c (unescape_nonascii): make dynamically compiled US-ASCII

include/ruby/ruby.h

@@ -230,7 +230,7 @@ typedef char ruby_check_sizeof_voidp[SIZEOF_VOIDP == sizeof(void*) ? 1 : -1];
 #define FIXNUM_MAX (LONG_MAX>>1)
 #define FIXNUM_MIN RSHIFT((long)LONG_MIN,1)
 
-#define INT2FIX(i) ((VALUE)(((SIGNED_VALUE)(i))<<1 | FIXNUM_FLAG))
+#define INT2FIX(i) (((VALUE)(i))<<1 | FIXNUM_FLAG)
 #define LONG2FIX(i) INT2FIX(i)
 #define rb_fix_new(v) INT2FIX(v)
 VALUE rb_int2inum(SIGNED_VALUE);

node.h

@@ -287,7 +287,7 @@ typedef struct RNode {
 #define NODE_LMASK  (((SIGNED_VALUE)1<<(sizeof(VALUE)*CHAR_BIT-NODE_LSHIFT))-1)
 #define nd_line(n) (int)(RNODE(n)->flags>>NODE_LSHIFT)
 #define nd_set_line(n,l) \
-    RNODE(n)->flags=((RNODE(n)->flags&~(-1<<NODE_LSHIFT))|(((l)&NODE_LMASK)<<NODE_LSHIFT))
+    RNODE(n)->flags=((RNODE(n)->flags&~((VALUE)(-1)<<NODE_LSHIFT))|((VALUE)((l)&NODE_LMASK)<<NODE_LSHIFT))
 
 #define nd_refinements  nd_reserved
 

pack.c

@@ -943,13 +943,14 @@ static const char b64_table[] =
 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 static void
-encodes(VALUE str, const char *s, long len, int type, int tail_lf)
+encodes(VALUE str, const char *s0, long len, int type, int tail_lf)
 {
     enum {buff_size = 4096, encoded_unit = 4, input_unit = 3};
     char buff[buff_size + 1];	/* +1 for tail_lf */
     long i = 0;
     const char *const trans = type == 'u' ? uu_table : b64_table;
     char padding;
+    const unsigned char *s = (const unsigned char *)s0;
 
     if (type == 'u') {
 	buff[i++] = (char)len + ' ';
@@ -1362,7 +1363,7 @@ pack_unpack(VALUE str, VALUE fmt)
 		t = RSTRING_PTR(bitstr);
 		for (i=0; i<len; i++) {
 		    if (i & 7) bits <<= 1;
-		    else bits = *s++;
+		    else bits = (unsigned char)*s++;
 		    *t++ = (bits & 128) ? '1' : '0';
 		}
 	    }
@@ -1406,7 +1407,7 @@ pack_unpack(VALUE str, VALUE fmt)
 		    if (i & 1)
 			bits <<= 4;
 		    else
-			bits = *s++;
+			bits = (unsigned char)*s++;
 		    *t++ = hexdigits[(bits >> 4) & 15];
 		}
 	    }

regint.h

@@ -392,7 +392,7 @@ typedef unsigned int  BitStatusType;
 #define BIT_STATUS_CLEAR(stats)      (stats) = 0
 #define BIT_STATUS_ON_ALL(stats)     (stats) = ~((BitStatusType )0)
 #define BIT_STATUS_AT(stats,n) \
-  ((n) < (int )BIT_STATUS_BITS_NUM  ?  ((stats) & (1 << n)) : ((stats) & 1))
+  ((n) < (int )BIT_STATUS_BITS_NUM  ?  ((stats) & ((BitStatusType)1 << n)) : ((stats) & 1))
 
 #define BIT_STATUS_ON_AT(stats,n) do {\
     if ((n) < (int )BIT_STATUS_BITS_NUM)	\
@@ -468,7 +468,7 @@ typedef Bits*          BitSetRef;
 } while (0)
 
 #define BS_ROOM(bs,pos)            (bs)[(int )(pos) / BITS_IN_ROOM]
-#define BS_BIT(pos)                (1 << ((int )(pos) % BITS_IN_ROOM))
+#define BS_BIT(pos)                (1U << ((int )(pos) % BITS_IN_ROOM))
 
 #define BITSET_AT(bs, pos)         (BS_ROOM(bs,pos) & BS_BIT(pos))
 #define BITSET_SET_BIT(bs, pos)     BS_ROOM(bs,pos) |= BS_BIT(pos)

time.c

@@ -4653,7 +4653,7 @@ time_mdump(VALUE time)
 	(vtm.mon-1)      << 10 | /*  4 */
 	vtm.mday         <<  5 | /*  5 */
 	vtm.hour;                /*  5 */
-    s = vtm.min          << 26 | /*  6 */
+    s = (unsigned long)vtm.min << 26 | /*  6 */
 	vtm.sec          << 20 | /*  6 */
 	usec;    /* 20 */
 
@@ -4766,10 +4766,10 @@ time_mload(VALUE time, VALUE str)
 
     p = s = 0;
     for (i=0; i<4; i++) {
-	p |= buf[i]<<(8*i);
+	p |= (unsigned long)buf[i]<<(8*i);
     }
     for (i=4; i<8; i++) {
-	s |= buf[i]<<(8*(i-4));
+	s |= (unsigned long)buf[i]<<(8*(i-4));
     }
 
     if ((p & (1UL<<31)) == 0) {

vm_core.h

@@ -811,7 +811,7 @@ enum vm_special_object_type {
 #define VM_FRAME_MAGIC_LAMBDA 0xa1
 #define VM_FRAME_MAGIC_RESCUE 0xb1
 #define VM_FRAME_MAGIC_MASK_BITS 8
-#define VM_FRAME_MAGIC_MASK   (~(~0<<VM_FRAME_MAGIC_MASK_BITS))
+#define VM_FRAME_MAGIC_MASK   (~(~(VALUE)0<<VM_FRAME_MAGIC_MASK_BITS))
 
 #define VM_FRAME_TYPE(cfp) ((cfp)->flag & VM_FRAME_MAGIC_MASK)
 

vm_trace.c

@@ -67,7 +67,7 @@ recalc_add_ruby_vm_event_flags(rb_event_flag_t events)
     ruby_vm_event_flags = 0;
 
     for (i=0; i<MAX_EVENT_NUM; i++) {
-	if (events & (1 << i)) {
+	if (events & ((rb_event_flag_t)1 << i)) {
 	    ruby_event_flag_count[i]++;
 	}
 	ruby_vm_event_flags |= ruby_event_flag_count[i] ? (1<<i) : 0;