gopls/internal/settings: enable semantic tokens by default

Whether or not semantic tokens are enabled should be a client-side
setting. Clients that don't want semantic tokens should not ask for
them, and if clients send textDocument/semanticTokens requests, gopls
should handle them by default. Unfortunately, as described in
golang/vscode-go#3338, this is not how semantic tokens were configured
in the past.

This CL changes the default value of the "semanticTokens" setting to
true. We should not release a gopls version with this change until
addressing golang/vscode-go#3338, but by making this change in master
now it is easier to test the fix in vscode-go.

For golang/vscode-go#3338

Change-Id: I05d7084436cd4dfe312460cfe08e3b1777f190ed
Reviewed-on: https://go-review.googlesource.com/c/tools/+/579337
Auto-Submit: Robert Findley <rfindley@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Alan Donovan <adonovan@google.com>

gopls/doc/release/v0.16.0.md

@@ -4,6 +4,13 @@ gopls/v0.16.0
 go install golang.org/x/tools/gopls@v0.16.0
 ```
 
+## Configuration Changes
+
+- The default value of the "semanticTokens" setting is now "true". This means
+  that if your LSP client is able and configured to request semantic tokens,
+  gopls will provide them. The default was previously false because VS Code
+  historically provided no client-side way for users to disable the feature.
+
 ## New features
 
 ### Integrated documentation viewer

gopls/doc/settings.md

@@ -196,9 +196,10 @@ Default: `{"gc_details":false,"generate":true,"regenerate_cgo":true,"tidy":true,
 **This setting is experimental and may be deleted.**
 
 semanticTokens controls whether the LSP server will send
-semantic tokens to the client.
+semantic tokens to the client. If false, gopls will send empty semantic
+tokens.
 
-Default: `false`.
+Default: `true`.
 
 #### **noSemanticString** *bool*
 

gopls/internal/settings/api_json.go

@@ -675,8 +675,8 @@ var GeneratedAPIJSON = &APIJSON{
 			{
 				Name:      "semanticTokens",
 				Type:      "bool",
-				Doc:       "semanticTokens controls whether the LSP server will send\nsemantic tokens to the client.\n",
-				Default:   "false",
+				Doc:       "semanticTokens controls whether the LSP server will send\nsemantic tokens to the client. If false, gopls will send empty semantic\ntokens.\n",
+				Default:   "true",
 				Status:    "experimental",
 				Hierarchy: "ui",
 			},

gopls/internal/settings/default.go

@@ -106,6 +106,7 @@ func DefaultOptions(overrides ...func(*Options)) *Options {
 						string(command.Vendor):            true,
 						// TODO(hyangah): enable command.RunGovulncheck.
 					},
+					SemanticTokens: true,
 				},
 			},
 			InternalOptions: InternalOptions{

gopls/internal/settings/settings.go

@@ -181,7 +181,8 @@ type UIOptions struct {
 	Codelenses map[string]bool
 
 	// SemanticTokens controls whether the LSP server will send
-	// semantic tokens to the client.
+	// semantic tokens to the client. If false, gopls will send empty semantic
+	// tokens.
 	SemanticTokens bool `status:"experimental"`
 
 	// NoSemanticString turns off the sending of the semantic token 'string'

gopls/internal/test/integration/misc/semantictokens_test.go

@@ -90,7 +90,6 @@ func Add[T int](target T, l []T) []T {
 `
 	WithOptions(
 		Modes(Default),
-		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		env.AfterChange(
@@ -127,7 +126,6 @@ func New[K int, V any]() Smap[K, V] {
 `
 	WithOptions(
 		Modes(Default),
-		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")
@@ -183,7 +181,6 @@ func bar() {}
 
 	WithOptions(
 		Modes(Default),
-		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")
@@ -198,7 +195,7 @@ func TestSemantic_65254(t *testing.T) {
 	src := `
 -- go.mod --
 module example.com
-	
+
 go 1.21
 -- main.go --
 package main
@@ -227,7 +224,6 @@ const bad = `
 	}
 	WithOptions(
 		Modes(Default),
-		Settings{"semanticTokens": true},
 	).Run(t, src, func(t *testing.T, env *Env) {
 		env.OpenFile("main.go")
 		seen := env.SemanticTokensFull("main.go")

gopls/internal/test/integration/template/template_test.go

@@ -36,7 +36,6 @@ go 1.17
 	WithOptions(
 		Settings{
 			"templateExtensions": []string{"tmpl"},
-			"semanticTokens":     true,
 		},
 	).Run(t, files, func(t *testing.T, env *Env) {
 		var p protocol.SemanticTokensParams
@@ -65,7 +64,6 @@ Hello {{}} <-- missing body
 	WithOptions(
 		Settings{
 			"templateExtensions": []string{"tmpl"},
-			"semanticTokens":     true,
 		},
 	).Run(t, files, func(t *testing.T, env *Env) {
 		// TODO: can we move this diagnostic onto {{}}?

gopls/internal/test/marker/testdata/token/comment.txt

@@ -5,11 +5,6 @@ links and output tokens according to the referenced object types, so that the
 editor can highlight them. This will help in checking the doc link errors and
 reading comments in the code.
 
--- settings.json --
-{
-	"semanticTokens": true
-}
-
 -- a.go --
 package p
 

gopls/internal/test/marker/testdata/token/range.txt

@@ -2,11 +2,6 @@ This test checks the output of textDocument/semanticTokens/range.
 
 TODO: add more assertions.
 
--- settings.json --
-{
-	"semanticTokens": true
-}
-
 -- a.go --
 package p //@token("package", "keyword", "")