test: fix flaky GoAwayThenClose (#4394)

In this test, we
1. make a streaming RPC on a connection
1. graceful stop it to send a GOAWAY
1. hard stop it, so the client will create a connection to another server

Before this fix, 2 and 3 can happen too soon, so the RPC in 1 would fail
and then transparent retry (because the stream is unprocessed by the
server in that case).

This retry attempt could pick the new connection, and then the RPC would
block until timeout.

After this streaming RPC fails, we make unary RPCs with the same
deadline (note: deadline not timeout) as the streaming RPC and expect
them to succeed. But they will also fail due to timeout.

The fix is to make a round-trip on the streaming RPC first, to make sure
it actually goes on the first connection.

test/end2end_test.go

@@ -6921,6 +6921,10 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 			return &testpb.SimpleResponse{}, nil
 		},
 		fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error {
+			if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
+				t.Errorf("unexpected error from send: %v", err)
+				return err
+			}
 			// Wait forever.
 			_, err := stream.Recv()
 			if err == nil {
@@ -6954,12 +6958,19 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 
 	client := testpb.NewTestServiceClient(cc)
 
-	// Should go on connection 1. We use a long-lived RPC because it will cause GracefulStop to send GO_AWAY, but the
-	// connection doesn't get closed until the server stops and the client receives.
+	// We make a streaming RPC and do an one-message-round-trip to make sure
+	// it's created on connection 1.
+	//
+	// We use a long-lived RPC because it will cause GracefulStop to send
+	// GO_AWAY, but the connection doesn't get closed until the server stops and
+	// the client receives the error.
 	stream, err := client.FullDuplexCall(ctx)
 	if err != nil {
 		t.Fatalf("FullDuplexCall(_) = _, %v; want _, nil", err)
 	}
+	if _, err = stream.Recv(); err != nil {
+		t.Fatalf("unexpected error from first recv: %v", err)
+	}
 
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{
 		{Addr: lis1.Addr().String()},
@@ -6976,8 +6987,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	s1.Stop()
 
 	// Wait for client to close.
-	_, err = stream.Recv()
-	if err == nil {
+	if _, err = stream.Recv(); err == nil {
 		t.Fatal("expected the stream to die, but got a successful Recv")
 	}