internal/modindex: add symbol information

This CL adds information about exported symbols to the module cache
index, together with a minimal test.

A future CL will add a Lookup function for finding completions for
selectors.

Change-Id: Ic36b5f30383187f9c5551ee2757258ed1445ea53
Reviewed-on: https://go-review.googlesource.com/c/tools/+/616416
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Robert Findley <rfindley@google.com>

internal/modindex/dir_test.go

@@ -37,7 +37,7 @@ var idtests = []id{
 			"cloud.google.com/go@v0.94.0/compute/metadata",
 		},
 	},
-	{ //m test bizarre characters in directory name
+	{ // test bizarre characters in directory name
 		importPath: "bad,guy.com/go",
 		best:       0,
 		dirs:       []string{"bad,guy.com/go@v0.1.0"},
@@ -51,17 +51,18 @@ func testModCache(t *testing.T) string {
 	return dir
 }
 
+// choose the semantically-latest version, with a single symbol
 func TestDirsSinglePath(t *testing.T) {
 	for _, itest := range idtests {
 		t.Run(itest.importPath, func(t *testing.T) {
-			// create a new fake GOMODCACHE
+			// create a new test GOMODCACHE
 			dir := testModCache(t)
 			for _, d := range itest.dirs {
 				if err := os.MkdirAll(filepath.Join(dir, d), 0755); err != nil {
 					t.Fatal(err)
 				}
-				// gopathwalk wants to see .go files
-				err := os.WriteFile(filepath.Join(dir, d, "main.go"), []byte("package main\nfunc main() {}"), 0600)
+				err := os.WriteFile(filepath.Join(dir, d, "foo.go"),
+					[]byte("package foo\nfunc Foo() {}"), 0600)
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -83,6 +84,13 @@ func TestDirsSinglePath(t *testing.T) {
 			if ix.Entries[0].Dir != Relpath(itest.dirs[itest.best]) {
 				t.Fatalf("got dir %s, wanted %s", ix.Entries[0].Dir, itest.dirs[itest.best])
 			}
+			nms := ix.Entries[0].Names
+			if len(nms) != 1 {
+				t.Fatalf("got %d names, expected 1", len(nms))
+			}
+			if nms[0] != "Foo F 0" {
+				t.Fatalf("got %q, expected Foo F 0", nms[0])
+			}
 		})
 	}
 }

internal/modindex/directories.go

@@ -23,6 +23,7 @@ type directory struct {
 	path       Relpath
 	importPath string
 	version    string // semantic version
+	syms       []symbol
 }
 
 // filterDirs groups the directories by import path,

internal/modindex/index.go

@@ -207,11 +207,13 @@ func writeIndex(cachedir Abspath, ix *Index) error {
 }
 
 func writeIndexToFile(x *Index, fd *os.File) error {
+	cnt := 0
 	w := bufio.NewWriter(fd)
 	fmt.Fprintf(w, "%d\n", x.Version)
 	fmt.Fprintf(w, "%s\n", x.Cachedir)
-	// TODO(pjw): round the time down
-	fmt.Fprintf(w, "%s\n", x.Changed.Format(time.DateTime))
+	// round the time down
+	tm := x.Changed.Add(-time.Second / 2)
+	fmt.Fprintf(w, "%s\n", tm.Format(time.DateTime))
 	for _, e := range x.Entries {
 		if e.ImportPath == "" {
 			continue // shouldn't happen
@@ -227,11 +229,13 @@ func writeIndexToFile(x *Index, fd *os.File) error {
 		}
 		for _, x := range e.Names {
 			fmt.Fprintf(w, "%s\n", x)
+			cnt++
 		}
 	}
 	if err := w.Flush(); err != nil {
 		return err
 	}
+	log.Printf("%d Entries %d names", len(x.Entries), cnt)
 	return nil
 }
 

internal/modindex/modindex.go

@@ -91,7 +91,7 @@ func (w *work) buildIndex() error {
 	if err != nil {
 		return err
 	}
-	log.Printf("%d dirs, %d ips", len(dirs), len(newdirs))
+	log.Printf("%d dirs, %d new", len(dirs), len(newdirs))
 	// for each import path it might occur only in newdirs,
 	// only in w.oldIndex, or in both.
 	// If it occurs in both, use the semantically later one
@@ -111,7 +111,7 @@ func (w *work) buildIndex() error {
 				delete(newdirs, e.ImportPath)
 			}
 		}
-		log.Printf("%d killed, %d ips", killed, len(newdirs))
+		log.Printf("%d killed, %d new", killed, len(newdirs))
 	}
 	// Build the skeleton of the new index using newdirs,
 	// and include the surviving parts of the old index
@@ -122,17 +122,24 @@ func (w *work) buildIndex() error {
 			}
 		}
 	}
+	// get symbol information for all the new diredtories
+	getSymbols(w.cacheDir, newdirs)
+	// assemble the new index entries
 	for k, v := range newdirs {
 		d := v[0]
+		pkg, names := processSyms(d.syms)
+		if pkg == "" {
+			continue // PJW: does this ever happen?
+		}
 		entry := Entry{
+			PkgName:    pkg,
 			Dir:        d.path,
 			ImportPath: k,
 			Version:    d.version,
+			Names:      names,
 		}
 		w.newIndex.Entries = append(w.newIndex.Entries, entry)
 	}
-	// find symbols for the incomplete entries
-	log.Print("not finding any symbols yet")
 	// sort the entries in the new index
 	slices.SortFunc(w.newIndex.Entries, func(l, r Entry) int {
 		if n := strings.Compare(l.PkgName, r.PkgName); n != 0 {

internal/modindex/symbols.go

@@ -0,0 +1,184 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package modindex
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"go/types"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// The name of a symbol contains information about the symbol:
+// <name> T for types
+// <name> C for consts
+// <name> V for vars
+// and for funcs: <name> F <num of return values> (<arg-name> <arg-type>)*
+// any spaces in <arg-type> are replaced by $s so that the fields
+// of the name are space separated
+type symbol struct {
+	pkg  string // name of the symbols's package
+	name string // declared name
+	kind string // T, C, V, or F
+	sig  string // signature information, for F
+}
+
+// find the symbols for the best directories
+func getSymbols(cd Abspath, dirs map[string][]*directory) {
+	var g errgroup.Group
+	g.SetLimit(-1) // maybe throttle this some day
+	for _, vv := range dirs {
+		// throttling some day?
+		d := vv[0]
+		g.Go(func() error {
+			thedir := filepath.Join(string(cd), string(d.path))
+			mode := parser.SkipObjectResolution
+
+			fi, err := os.ReadDir(thedir)
+			if err != nil {
+				return nil // log this someday?
+			}
+			for _, fx := range fi {
+				if !strings.HasSuffix(fx.Name(), ".go") || strings.HasSuffix(fx.Name(), "_test.go") {
+					continue
+				}
+				fname := filepath.Join(thedir, fx.Name())
+				tr, err := parser.ParseFile(token.NewFileSet(), fname, nil, mode)
+				if err != nil {
+					continue // ignore errors, someday log them?
+				}
+				d.syms = append(d.syms, getFileExports(tr)...)
+			}
+			return nil
+		})
+	}
+	g.Wait()
+}
+
+func getFileExports(f *ast.File) []symbol {
+	pkg := f.Name.Name
+	if pkg == "main" {
+		return nil
+	}
+	var ans []symbol
+	// should we look for //go:build ignore?
+	for _, decl := range f.Decls {
+		switch decl := decl.(type) {
+		case *ast.FuncDecl:
+			if decl.Recv != nil {
+				// ignore methods, as we are completing package selections
+				continue
+			}
+			name := decl.Name.Name
+			dtype := decl.Type
+			// not looking at dtype.TypeParams. That is, treating
+			// generic functions just like non-generic ones.
+			sig := dtype.Params
+			kind := "F"
+			result := []string{fmt.Sprintf("%d", dtype.Results.NumFields())}
+			for _, x := range sig.List {
+				// This code creates a string representing the type.
+				// TODO(pjw): it may be fragile:
+				// 1. x.Type could be nil, perhaps in ill-formed code
+				// 2. ExprString might someday change incompatibly to
+				//    include struct tags, which can be arbitrary strings
+				if x.Type == nil {
+					// Can this happen without a parse error? (Files with parse
+					// errors are ignored in getSymbols)
+					continue // maybe report this someday
+				}
+				tp := types.ExprString(x.Type)
+				if len(tp) == 0 {
+					// Can this happen?
+					continue // maybe report this someday
+				}
+				// This is only safe if ExprString never returns anything with a $
+				// The only place a $ can occur seems to be in a struct tag, which
+				// can be an arbitrary string literal, and ExprString does not presently
+				// print struct tags. So for this to happen the type of a formal parameter
+				// has to be a explict struct, e.g. foo(x struct{a int "$"}) and ExprString
+				// would have to show the struct tag. Even testing for this case seems
+				// a waste of effort, but let's not ignore such pathologies
+				if strings.Contains(tp, "$") {
+					continue
+				}
+				tp = strings.Replace(tp, " ", "$", -1)
+				for _, y := range x.Names {
+					result = append(result, y.Name)
+					result = append(result, tp)
+				}
+			}
+			sigs := strings.Join(result, " ")
+			if s := newsym(pkg, name, kind, sigs); s != nil {
+				ans = append(ans, *s)
+			}
+		case *ast.GenDecl:
+			switch decl.Tok {
+			case token.CONST, token.VAR:
+				tp := " V"
+				if decl.Tok == token.CONST {
+					tp = " C"
+				}
+				for _, sp := range decl.Specs {
+					for _, x := range sp.(*ast.ValueSpec).Names {
+						if s := newsym(pkg, x.Name, tp, ""); s != nil {
+							ans = append(ans, *s)
+						}
+					}
+				}
+			case token.TYPE:
+				for _, sp := range decl.Specs {
+					if s := newsym(pkg, sp.(*ast.TypeSpec).Name.Name, " T", ""); s != nil {
+						ans = append(ans, *s)
+					}
+				}
+			}
+		}
+	}
+	return ans
+}
+
+func newsym(pkg, name, kind, sig string) *symbol {
+	if len(name) == 0 || !ast.IsExported(name) {
+		return nil
+	}
+	sym := symbol{pkg: pkg, name: name, kind: kind, sig: sig}
+	return &sym
+}
+
+// return the package name and the value for the symbols.
+// if there are multiple packages, choose one arbitrarily
+// the returned slice is sorted lexicographically
+func processSyms(syms []symbol) (string, []string) {
+	if len(syms) == 0 {
+		return "", nil
+	}
+	slices.SortFunc(syms, func(l, r symbol) int {
+		return strings.Compare(l.name, r.name)
+	})
+	pkg := syms[0].pkg
+	var names []string
+	for _, s := range syms {
+		var nx string
+		if s.pkg == pkg {
+			if s.sig != "" {
+				nx = fmt.Sprintf("%s %s %s", s.name, s.kind, s.sig)
+			} else {
+				nx = fmt.Sprintf("%s %s", s.name, s.kind)
+			}
+			names = append(names, nx)
+		} else {
+			continue // PJW: do we want to keep track of these?
+		}
+	}
+	return pkg, names
+}