go/types/typeutil: don't recurse into constraints when hashing tparams

When hashing signature type parameters, we recursed into their
constraints in an attempt to improve the accuracy of the hash
(signatures are considered identical modulo type parameter renaming).

This introduces a mechanism for infinite recursion via recursive
constraints, as reported in golang/go#51314. To fix this, we can hash
just the type parameter index and rely on types.Identical for
de-duplicating map entries.

Fixes golang/go#51314

Change-Id: Ifd7596d978b98293391bbe64f72f1da2b2a5d51c
Reviewed-on: https://go-review.googlesource.com/c/tools/+/387754
Trust: Robert Findley <rfindley@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
Reviewed-by: Tim King <taking@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

go/types/typeutil/map.go

@@ -379,7 +379,7 @@ func (h Hasher) hashFor(t types.Type) uint32 {
 func (h Hasher) hashTuple(tuple *types.Tuple) uint32 {
 	// See go/types.identicalTypes for rationale.
 	n := tuple.Len()
-	var hash uint32 = 9137 + 2*uint32(n)
+	hash := 9137 + 2*uint32(n)
 	for i := 0; i < n; i++ {
 		hash += 3 * h.Hash(tuple.At(i).Type())
 	}
@@ -398,7 +398,7 @@ func (h Hasher) hashUnion(t *typeparams.Union) uint32 {
 }
 
 func (h Hasher) hashTermSet(terms []*typeparams.Term) uint32 {
-	var hash uint32 = 9157 + 2*uint32(len(terms))
+	hash := 9157 + 2*uint32(len(terms))
 	for _, term := range terms {
 		// term order is not significant.
 		termHash := h.Hash(term.Type())
@@ -416,14 +416,16 @@ func (h Hasher) hashTermSet(terms []*typeparams.Term) uint32 {
 // If h.sigTParams is set and contains t, then we are in the process of hashing
 // a signature, and the hash value of t must depend only on t's index and
 // constraint: signatures are considered identical modulo type parameter
-// renaming.
+// renaming. To avoid infinite recursion, we only hash the type parameter
+// index, and rely on types.Identical to handle signatures where constraints
+// are not identical.
 //
 // Otherwise the hash of t depends only on t's pointer identity.
 func (h Hasher) hashTypeParam(t *typeparams.TypeParam) uint32 {
 	if h.sigTParams != nil {
 		i := t.Index()
 		if i >= 0 && i < h.sigTParams.Len() && t == h.sigTParams.At(i) {
-			return 9173 + 2*h.Hash(t.Constraint()) + 3*uint32(i)
+			return 9173 + 3*uint32(i)
 		}
 	}
 	return h.hashPtr(t.Obj())

go/types/typeutil/map_test.go

@@ -233,6 +233,17 @@ var ME2 = G2[int].M
 
 // ME1Type should have identical type as ME1.
 var ME1Type func(G1[int], G1[int], G2[int])
+
+// Examples from issue #51314
+type Constraint[T any] interface{}
+func Foo[T Constraint[T]]() {}
+func Fn[T1 ~*T2, T2 ~*T1](t1 T1, t2 T2) {}
+
+// Bar and Baz are identical to Foo.
+func Bar[P Constraint[P]]() {}
+func Baz[Q any]() {} // The underlying type of Constraint[P] is any.
+// But Quux is not.
+func Quux[Q interface{ quux() }]() {}
 `
 
 	fset := token.NewFileSet()
@@ -284,6 +295,13 @@ var ME1Type func(G1[int], G1[int], G2[int])
 		ME1     = scope.Lookup("ME1").Type()
 		ME1Type = scope.Lookup("ME1Type").Type()
 		ME2     = scope.Lookup("ME2").Type()
+
+		Constraint = scope.Lookup("Constraint").Type()
+		Foo        = scope.Lookup("Foo").Type()
+		Fn         = scope.Lookup("Fn").Type()
+		Bar        = scope.Lookup("Foo").Type()
+		Baz        = scope.Lookup("Foo").Type()
+		Quux       = scope.Lookup("Quux").Type()
 	)
 
 	tmap := new(typeutil.Map)
@@ -345,6 +363,14 @@ var ME1Type func(G1[int], G1[int], G2[int])
 		{ME1, "ME1", true},
 		{ME1Type, "ME1Type", false},
 		{ME2, "ME2", true},
+
+		// See golang/go#51314: avoid infinite recursion on cyclic type constraints.
+		{Constraint, "Constraint", true},
+		{Foo, "Foo", true},
+		{Fn, "Fn", true},
+		{Bar, "Bar", false},
+		{Baz, "Baz", false},
+		{Quux, "Quux", true},
 	}
 
 	for _, step := range steps {