x86/x86asm: fix three panics on malformed input

All three test cases used to cause a panic.

Change-Id: I2874c1fb6f09e6c3678d0b8d469f2c582dd8f50b
Reviewed-on: https://go-review.googlesource.com/c/155939
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

x86/x86asm/decode.go

@@ -203,7 +203,9 @@ func instPrefix(b byte, mode int) (Inst, error) {
 // For now we use instPrefix but perhaps later we will return
 // a specific error here.
 func truncated(src []byte, mode int) (Inst, error) {
-	//	return Inst{}, len(src), ErrTruncated
+	if len(src) == 0 {
+		return Inst{}, ErrTruncated
+	}
 	return instPrefix(src[0], mode) // too long
 }
 
@@ -406,7 +408,7 @@ ReadPrefixes:
 
 		//Group 5 - Vex encoding
 		case 0xC5:
-			if pos == 0 && (mode == 64 || (mode == 32 && pos+1 < len(src) && src[pos+1]&0xc0 == 0xc0)) {
+			if pos == 0 && pos+1 < len(src) && (mode == 64 || (mode == 32 && src[pos+1]&0xc0 == 0xc0)) {
 				vex = p
 				vexIndex = pos
 				inst.Prefix[pos] = p
@@ -418,7 +420,7 @@ ReadPrefixes:
 				break ReadPrefixes
 			}
 		case 0xC4:
-			if pos == 0 && (mode == 64 || (mode == 32 && pos+2 < len(src) && src[pos+1]&0xc0 == 0xc0)) {
+			if pos == 0 && pos+2 < len(src) && (mode == 64 || (mode == 32 && src[pos+1]&0xc0 == 0xc0)) {
 				vex = p
 				vexIndex = pos
 				inst.Prefix[pos] = p

x86/x86asm/decode_test.go

@@ -69,3 +69,17 @@ func TestDecode(t *testing.T) {
 		}
 	}
 }
+
+func TestDecodeDoesNotCrash(t *testing.T) {
+	cases := [...][]byte{
+		[]byte{},
+		[]byte{0xc5},
+		[]byte{0xc4},
+	}
+	for _, test := range cases {
+		_, err := Decode([]byte(test), 64) // the only goal is that this line does not panic
+		if err == nil {
+			t.Errorf("expected error on invalid instruction %x", test)
+		}
+	}
+}