Only perform the disp32->disp8 addressing mode optimization on valid instructions (bug 543440 r=dvander+)

RIP-relative addressing uses mod 00 "disp32" encoding, but mod_disp32() assumes
all instructions passed in can be optimized to disp8 encoding if the actual
displacement is 8-bit.  This is invalid for mode 00 modes, including RIP addressing.

mod_disp32() can still do the right thing in the other arm of its branch; this
patch removes the assert and tightens the check for the disp8 case.

--HG--
extra : convert_revision : b534b9289ca8a111f16fbf0b0711b5277be440f4

js/src/nanojit/NativeX64.cpp

@@ -171,9 +171,9 @@ namespace nanojit
     static inline uint64_t mod_disp32(uint64_t op, Register r, Register b, int32_t d) {
         NanoAssert(IsGpReg(r) && IsGpReg(b));
         NanoAssert((b & 7) != 4); // using RSP or R12 as base requires SIB
-        if (isS8(d)) {
+        uint64_t mod = (((op>>24)&255)>>6); // mod bits in addressing mode: 0,1,2, or 3
+        if (mod == 2 && isS8(d)) {
             // op is:  0x[disp32=0][mod=2:r:b][op][rex][len]
-            NanoAssert((((op>>24)&255)>>6) == 2); // disp32 mode
             int len = oplen(op);
             op = (op & ~0xff000000LL) | (0x40 | (r&7)<<3 | (b&7))<<24; // replace mod
             return op<<24 | int64_t(d)<<56 | (len-3); // shrink disp, add disp8