internal/lsp: make source independent of protocol

I realized this was a mistake, we should try to keep the source
directory independent of the LSP protocol itself, and adapt in
the outer layer.
This will keep us honest about capabilities, let us add the
caching and conversion layers easily, and also allow for a future
where we expose the source directory as a supported API for other
tools.
The outer lsp package then becomes the adapter from the core
features to the specifics of the LSP protocol.

Change-Id: I68fd089f1b9f2fd38decc1cbc13c6f0f86157b94
Reviewed-on: https://go-review.googlesource.com/c/148157
Reviewed-by: Rebecca Stambler <rstambler@golang.org>

internal/lsp/completion.go

@@ -15,15 +15,25 @@ import (
 )
 
 func completion(v *source.View, uri protocol.DocumentURI, pos protocol.Position) (items []protocol.CompletionItem, err error) {
-	pkg, qfile, qpos, err := v.TypeCheckAtPosition(uri, pos)
+	f := v.GetFile(source.URI(uri))
 	if err != nil {
 		return nil, err
 	}
-	items, _, err = completions(pkg.Fset, qfile, qpos, pkg.Types, pkg.TypesInfo)
+	tok, err := f.GetToken()
 	if err != nil {
 		return nil, err
 	}
-	return items, nil
+	p := fromProtocolPosition(tok, pos)
+	file, err := f.GetAST() // Use p to prune the AST?
+	if err != nil {
+		return nil, err
+	}
+	pkg, err := f.GetPackage()
+	if err != nil {
+		return nil, err
+	}
+	items, _, err = completions(v.Config.Fset, file, p, pkg.Types, pkg.TypesInfo)
+	return items, err
 }
 
 // Completions returns the map of possible candidates for completion,

internal/lsp/diagnostics.go

@@ -14,8 +14,8 @@ import (
 	"golang.org/x/tools/internal/lsp/source"
 )
 
-func diagnostics(v *source.View, uri protocol.DocumentURI) (map[string][]protocol.Diagnostic, error) {
-	pkg, err := v.TypeCheck(uri)
+func diagnostics(v *source.View, uri source.URI) (map[string][]protocol.Diagnostic, error) {
+	pkg, err := v.GetFile(uri).GetPackage()
 	if err != nil {
 		return nil, err
 	}

internal/lsp/diagnostics_test.go

@@ -86,8 +86,11 @@ func testDiagnostics(t *testing.T, exporter packagestest.Exporter) {
 		t.Fatal(err)
 	}
 	v := source.NewView()
-	v.Config = exported.Config
-	v.Config.Mode = packages.LoadSyntax
+	// merge the config objects
+	cfg := *exported.Config
+	cfg.Fset = v.Config.Fset
+	cfg.Mode = packages.LoadSyntax
+	v.Config = &cfg
 	for _, pkg := range pkgs {
 		for _, filename := range pkg.GoFiles {
 			diagnostics, err := diagnostics(v, source.ToURI(filename))

internal/lsp/format.go

@@ -15,7 +15,7 @@ import (
 
 // formatRange formats a document with a given range.
 func formatRange(v *source.View, uri protocol.DocumentURI, rng *protocol.Range) ([]protocol.TextEdit, error) {
-	data, err := v.GetFile(uri).Read()
+	data, err := v.GetFile(source.URI(uri)).Read()
 	if err != nil {
 		return nil, err
 	}

internal/lsp/position.go

@@ -0,0 +1,109 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lsp
+
+import (
+	"go/token"
+
+	"golang.org/x/tools/internal/lsp/protocol"
+	"golang.org/x/tools/internal/lsp/source"
+)
+
+// fromProtocolLocation converts from a protocol location to a source range.
+// It will return an error if the file of the location was not valid.
+// It uses fromProtocolRange to convert the start and end positions.
+func fromProtocolLocation(v *source.View, loc protocol.Location) (source.Range, error) {
+	f := v.GetFile(source.URI(loc.URI))
+	tok, err := f.GetToken()
+	if err != nil {
+		return source.Range{}, err
+	}
+	return fromProtocolRange(tok, loc.Range), nil
+}
+
+// toProtocolLocation converts from a source range back to a protocol location.
+func toProtocolLocation(v *source.View, r source.Range) protocol.Location {
+	tokFile := v.Config.Fset.File(r.Start)
+	file := v.GetFile(source.ToURI(tokFile.Name()))
+	return protocol.Location{
+		URI: protocol.DocumentURI(file.URI),
+		Range: protocol.Range{
+			Start: toProtocolPosition(tokFile, r.Start),
+			End:   toProtocolPosition(tokFile, r.End),
+		},
+	}
+}
+
+// fromProtocolRange converts a protocol range to a source range.
+// It uses fromProtocolPosition to convert the start and end positions, which
+// requires the token file the positions belongs to.
+func fromProtocolRange(f *token.File, r protocol.Range) source.Range {
+	start := fromProtocolPosition(f, r.Start)
+	var end token.Pos
+	switch {
+	case r.End == r.Start:
+		end = start
+	case r.End.Line < 0:
+		end = token.NoPos
+	default:
+		end = fromProtocolPosition(f, r.End)
+	}
+	return source.Range{
+		Start: start,
+		End:   end,
+	}
+}
+
+// fromProtocolPosition converts a protocol position (0-based line and column
+// number) to a token.Pos (byte offset value).
+// It requires the token file the pos belongs to in order to do this.
+func fromProtocolPosition(f *token.File, pos protocol.Position) token.Pos {
+	line := lineStart(f, int(pos.Line)+1)
+	return line + token.Pos(pos.Character) // TODO: this is wrong, bytes not characters
+}
+
+// toProtocolPosition converts from a token pos (byte offset) to a protocol
+// position  (0-based line and column number)
+// It requires the token file the pos belongs to in order to do this.
+func toProtocolPosition(f *token.File, pos token.Pos) protocol.Position {
+	if !pos.IsValid() {
+		return protocol.Position{Line: -1.0, Character: -1.0}
+	}
+	p := f.Position(pos)
+	return protocol.Position{
+		Line:      float64(p.Line - 1),
+		Character: float64(p.Column - 1),
+	}
+}
+
+// this functionality was borrowed from the analysisutil package
+func lineStart(f *token.File, line int) token.Pos {
+	// Use binary search to find the start offset of this line.
+	//
+	// TODO(adonovan): eventually replace this function with the
+	// simpler and more efficient (*go/token.File).LineStart, added
+	// in go1.12.
+
+	min := 0        // inclusive
+	max := f.Size() // exclusive
+	for {
+		offset := (min + max) / 2
+		pos := f.Pos(offset)
+		posn := f.Position(pos)
+		if posn.Line == line {
+			return pos - (token.Pos(posn.Column) - 1)
+		}
+
+		if min+1 >= max {
+			return token.NoPos
+		}
+
+		if posn.Line < line {
+			min = offset
+		} else {
+			max = offset
+		}
+	}
+}

internal/lsp/server.go

@@ -114,13 +114,14 @@ func (s *server) DidChange(ctx context.Context, params *protocol.DidChangeTextDo
 }
 
 func (s *server) cacheAndDiagnoseFile(ctx context.Context, uri protocol.DocumentURI, text string) {
-	s.view.GetFile(uri).SetContent([]byte(text))
+	f := s.view.GetFile(source.URI(uri))
+	f.SetContent([]byte(text))
 	go func() {
-		reports, err := diagnostics(s.view, uri)
+		reports, err := diagnostics(s.view, f.URI)
 		if err == nil {
 			for filename, diagnostics := range reports {
 				s.client.PublishDiagnostics(ctx, &protocol.PublishDiagnosticsParams{
-					URI:         source.ToURI(filename),
+					URI:         protocol.DocumentURI(source.ToURI(filename)),
 					Diagnostics: diagnostics,
 				})
 			}
@@ -142,7 +143,7 @@ func (s *server) DidSave(context.Context, *protocol.DidSaveTextDocumentParams) e
 }
 
 func (s *server) DidClose(ctx context.Context, params *protocol.DidCloseTextDocumentParams) error {
-	s.view.GetFile(params.TextDocument.URI).SetContent(nil)
+	s.view.GetFile(source.URI(params.TextDocument.URI)).SetContent(nil)
 	return nil
 }
 

internal/lsp/source/file.go

@@ -5,21 +5,31 @@
 package source
 
 import (
+	"fmt"
 	"go/ast"
 	"go/token"
+	"golang.org/x/tools/go/packages"
 	"io/ioutil"
-
-	"golang.org/x/tools/internal/lsp/protocol"
 )
 
 // File holds all the information we know about a file.
 type File struct {
-	URI     protocol.DocumentURI
+	URI     URI
 	view    *View
 	active  bool
 	content []byte
 	ast     *ast.File
 	token   *token.File
+	pkg     *packages.Package
+}
+
+// Range represents a start and end position.
+// Because Range is based purely on two token.Pos entries, it is not self
+// contained. You need access to a token.FileSet to regain the file
+// information.
+type Range struct {
+	Start token.Pos
+	End   token.Pos
 }
 
 // SetContent sets the overlay contents for a file.
@@ -32,19 +42,20 @@ func (f *File) SetContent(content []byte) {
 	// the ast and token fields are invalid
 	f.ast = nil
 	f.token = nil
+	f.pkg = nil
 	// and we might need to update the overlay
 	switch {
 	case f.active && content == nil:
 		// we were active, and want to forget the content
 		f.active = false
-		if filename, err := FromURI(f.URI); err == nil {
+		if filename, err := f.URI.Filename(); err == nil {
 			delete(f.view.Config.Overlay, filename)
 		}
 		f.content = nil
 	case content != nil:
 		// an active overlay, update the map
 		f.active = true
-		if filename, err := FromURI(f.URI); err == nil {
+		if filename, err := f.URI.Filename(); err == nil {
 			f.view.Config.Overlay[filename] = f.content
 		}
 	}
@@ -57,13 +68,49 @@ func (f *File) Read() ([]byte, error) {
 	return f.read()
 }
 
+func (f *File) GetToken() (*token.File, error) {
+	f.view.mu.Lock()
+	defer f.view.mu.Unlock()
+	if f.token == nil {
+		if err := f.view.parse(f.URI); err != nil {
+			return nil, err
+		}
+		if f.token == nil {
+			return nil, fmt.Errorf("failed to find or parse %v", f.URI)
+		}
+	}
+	return f.token, nil
+}
+
+func (f *File) GetAST() (*ast.File, error) {
+	f.view.mu.Lock()
+	defer f.view.mu.Unlock()
+	if f.ast == nil {
+		if err := f.view.parse(f.URI); err != nil {
+			return nil, err
+		}
+	}
+	return f.ast, nil
+}
+
+func (f *File) GetPackage() (*packages.Package, error) {
+	f.view.mu.Lock()
+	defer f.view.mu.Unlock()
+	if f.pkg == nil {
+		if err := f.view.parse(f.URI); err != nil {
+			return nil, err
+		}
+	}
+	return f.pkg, nil
+}
+
 // read is the internal part of Read that presumes the lock is already held
 func (f *File) read() ([]byte, error) {
 	if f.content != nil {
 		return f.content, nil
 	}
 	// we don't know the content yet, so read it
-	filename, err := FromURI(f.URI)
+	filename, err := f.URI.Filename()
 	if err != nil {
 		return nil, err
 	}

internal/lsp/source/uri.go

@@ -6,27 +6,35 @@ package source
 
 import (
 	"fmt"
+	"net/url"
 	"path/filepath"
 	"strings"
-
-	"golang.org/x/tools/internal/lsp/protocol"
 )
 
 const fileSchemePrefix = "file://"
 
-// FromURI gets the file path for a given URI.
+// URI represents the full uri for a file.
+type URI string
+
+// Filename gets the file path for the URI.
 // It will return an error if the uri is not valid, or if the URI was not
 // a file URI
-func FromURI(uri protocol.DocumentURI) (string, error) {
+func (uri URI) Filename() (string, error) {
 	s := string(uri)
 	if !strings.HasPrefix(s, fileSchemePrefix) {
 		return "", fmt.Errorf("only file URI's are supported, got %v", uri)
 	}
-	return filepath.FromSlash(s[len(fileSchemePrefix):]), nil
+	s = s[len(fileSchemePrefix):]
+	s, err := url.PathUnescape(s)
+	if err != nil {
+		return s, err
+	}
+	s = filepath.FromSlash(s)
+	return s, nil
 }
 
 // ToURI returns a protocol URI for the supplied path.
 // It will always have the file scheme.
-func ToURI(path string) protocol.DocumentURI {
-	return protocol.DocumentURI(fileSchemePrefix + filepath.ToSlash(path))
+func ToURI(path string) URI {
+	return URI(fileSchemePrefix + filepath.ToSlash(path))
 }

internal/lsp/source/view.go

@@ -5,17 +5,11 @@
 package source
 
 import (
-	"bytes"
 	"fmt"
-	"go/ast"
-	"go/parser"
 	"go/token"
-	"os"
-	"path/filepath"
 	"sync"
 
 	"golang.org/x/tools/go/packages"
-	"golang.org/x/tools/internal/lsp/protocol"
 )
 
 type View struct {
@@ -23,7 +17,7 @@ type View struct {
 
 	Config *packages.Config
 
-	files map[protocol.DocumentURI]*File
+	files map[URI]*File
 }
 
 func NewView() *View {
@@ -34,14 +28,21 @@ func NewView() *View {
 			Tests:   true,
 			Overlay: make(map[string][]byte),
 		},
-		files: make(map[protocol.DocumentURI]*File),
+		files: make(map[URI]*File),
 	}
 }
 
 // GetFile returns a File for the given uri.
 // It will always succeed, adding the file to the managed set if needed.
-func (v *View) GetFile(uri protocol.DocumentURI) *File {
+func (v *View) GetFile(uri URI) *File {
 	v.mu.Lock()
+	f := v.getFile(uri)
+	v.mu.Unlock()
+	return f
+}
+
+// getFile is the unlocked internal implementation of GetFile.
+func (v *View) getFile(uri URI) *File {
 	f, found := v.files[uri]
 	if !found {
 		f = &File{
@@ -50,166 +51,32 @@ func (v *View) GetFile(uri protocol.DocumentURI) *File {
 		}
 		v.files[f.URI] = f
 	}
-	v.mu.Unlock()
 	return f
 }
 
-// TypeCheck type-checks the package for the given package path.
-func (v *View) TypeCheck(uri protocol.DocumentURI) (*packages.Package, error) {
-	v.mu.Lock()
-	defer v.mu.Unlock()
-	path, err := FromURI(uri)
+func (v *View) parse(uri URI) error {
+	path, err := uri.Filename()
 	if err != nil {
-		return nil, err
+		return err
 	}
 	pkgs, err := packages.Load(v.Config, fmt.Sprintf("file=%s", path))
 	if len(pkgs) == 0 {
 		if err == nil {
 			err = fmt.Errorf("no packages found for %s", path)
 		}
-		return nil, err
+		return err
 	}
-	pkg := pkgs[0]
-	return pkg, nil
-}
-
-func (v *View) TypeCheckAtPosition(uri protocol.DocumentURI, position protocol.Position) (*packages.Package, *ast.File, token.Pos, error) {
-	v.mu.Lock()
-	defer v.mu.Unlock()
-	filename, err := FromURI(uri)
-	if err != nil {
-		return nil, nil, token.NoPos, err
-	}
-
-	var mu sync.Mutex
-	var qfileContent []byte
-
-	cfg := &packages.Config{
-		Mode:       v.Config.Mode,
-		Dir:        v.Config.Dir,
-		Env:        v.Config.Env,
-		BuildFlags: v.Config.BuildFlags,
-		Fset:       v.Config.Fset,
-		Tests:      v.Config.Tests,
-		Overlay:    v.Config.Overlay,
-		ParseFile: func(fset *token.FileSet, current string, data []byte) (*ast.File, error) {
-			// Save the file contents for use later in determining the query position.
-			if sameFile(current, filename) {
-				mu.Lock()
-				qfileContent = data
-				mu.Unlock()
-			}
-			return parser.ParseFile(fset, current, data, parser.AllErrors)
-		},
-	}
-	pkgs, err := packages.Load(cfg, fmt.Sprintf("file=%s", filename))
-	if len(pkgs) == 0 {
-		if err == nil {
-			err = fmt.Errorf("no package found for %s", filename)
-		}
-		return nil, nil, token.NoPos, err
-	}
-	pkg := pkgs[0]
-
-	var qpos token.Pos
-	var qfile *ast.File
-	for _, file := range pkg.Syntax {
-		tokfile := pkg.Fset.File(file.Pos())
-		if tokfile == nil || tokfile.Name() != filename {
-			continue
-		}
-		pos := positionToPos(tokfile, qfileContent, int(position.Line), int(position.Character))
-		if !pos.IsValid() {
-			return nil, nil, token.NoPos, fmt.Errorf("invalid position for %s", filename)
-		}
-		qfile = file
-		qpos = pos
-		break
-	}
-
-	if qfile == nil || qpos == token.NoPos {
-		return nil, nil, token.NoPos, fmt.Errorf("unable to find position %s:%v:%v", filename, position.Line, position.Character)
-	}
-	return pkg, qfile, qpos, nil
-}
-
-// trimAST clears any part of the AST not relevant to type checking
-// expressions at pos.
-func trimAST(file *ast.File, pos token.Pos) {
-	ast.Inspect(file, func(n ast.Node) bool {
-		if n == nil {
-			return false
-		}
-		if pos < n.Pos() || pos >= n.End() {
-			switch n := n.(type) {
-			case *ast.FuncDecl:
-				n.Body = nil
-			case *ast.BlockStmt:
-				n.List = nil
-			case *ast.CaseClause:
-				n.Body = nil
-			case *ast.CommClause:
-				n.Body = nil
-			case *ast.CompositeLit:
-				// Leave elts in place for [...]T
-				// array literals, because they can
-				// affect the expression's type.
-				if !isEllipsisArray(n.Type) {
-					n.Elts = nil
-				}
-			}
-		}
-		return true
-	})
-}
-
-func isEllipsisArray(n ast.Expr) bool {
-	at, ok := n.(*ast.ArrayType)
-	if !ok {
-		return false
-	}
-	_, ok = at.Len.(*ast.Ellipsis)
-	return ok
-}
-
-func sameFile(filename1, filename2 string) bool {
-	if filepath.Base(filename1) != filepath.Base(filename2) {
-		return false
-	}
-	finfo1, err := os.Stat(filename1)
-	if err != nil {
-		return false
-	}
-	finfo2, err := os.Stat(filename2)
-	if err != nil {
-		return false
-	}
-	return os.SameFile(finfo1, finfo2)
-}
-
-// positionToPos converts a 0-based line and column number in a file
-// to a token.Pos. It returns NoPos if the file did not contain the position.
-func positionToPos(file *token.File, content []byte, line, col int) token.Pos {
-	if file.Size() != len(content) {
-		return token.NoPos
-	}
-	if file.LineCount() < int(line) { // these can be equal if the last line is empty
-		return token.NoPos
-	}
-	start := 0
-	for i := 0; i < int(line); i++ {
-		if start >= len(content) {
-			return token.NoPos
-		}
-		index := bytes.IndexByte(content[start:], '\n')
-		if index == -1 {
-			return token.NoPos
-		}
-		start += (index + 1)
-	}
-	offset := start + int(col)
-	if offset > file.Size() {
-		return token.NoPos
-	}
-	return file.Pos(offset)
+	for _, pkg := range pkgs {
+		// add everything we find to the files cache
+		for _, fAST := range pkg.Syntax {
+			// if a file was in multiple packages, which token/ast/pkg do we store
+			fToken := v.Config.Fset.File(fAST.Pos())
+			fURI := ToURI(fToken.Name())
+			f := v.getFile(fURI)
+			f.token = fToken
+			f.ast = fAST
+			f.pkg = pkg
+		}
+	}
+	return nil
 }