openpgp/elgamal: prevent bad key from causing panic in Decrypt

If the mod inverse of the private key's P value does not exist,
return an error in Decrypt rather than panic.

Change-Id: Ia075a60108863b14ba98bb62364a17131423b819
Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/573976
Reviewed-by: Filippo Valsorda <valsorda@google.com>
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/205502
Run-TryBot: Katie Hockman <katie@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Filippo Valsorda <filippo@golang.org>

openpgp/elgamal/elgamal.go

@@ -76,7 +76,9 @@ func Encrypt(random io.Reader, pub *PublicKey, msg []byte) (c1, c2 *big.Int, err
 // Bleichenbacher, Advances in Cryptology (Crypto '98),
 func Decrypt(priv *PrivateKey, c1, c2 *big.Int) (msg []byte, err error) {
 	s := new(big.Int).Exp(c1, priv.X, priv.P)
-	s.ModInverse(s, priv.P)
+	if s.ModInverse(s, priv.P) == nil {
+		return nil, errors.New("elgamal: invalid private key")
+	}
 	s.Mul(s, c2)
 	s.Mod(s, priv.P)
 	em := s.Bytes()

openpgp/elgamal/elgamal_test.go

@@ -47,3 +47,18 @@ func TestEncryptDecrypt(t *testing.T) {
 		t.Errorf("decryption failed, got: %x, want: %x", message2, message)
 	}
 }
+
+func TestDecryptBadKey(t *testing.T) {
+	priv := &PrivateKey{
+		PublicKey: PublicKey{
+			G: fromHex(generatorHex),
+			P: fromHex("2"),
+		},
+		X: fromHex("42"),
+	}
+	priv.Y = new(big.Int).Exp(priv.G, priv.X, priv.P)
+	c1, c2 := fromHex("8"), fromHex("8")
+	if _, err := Decrypt(priv, c1, c2); err == nil {
+		t.Errorf("unexpected success decrypting")
+	}
+}