internal/cmd/deadcode: a command to report dead code in Go programs

This CL adds a command to report functions that are unreachable
from the main functions of applications and tests.

It uses the Rapid Type Analysis (RTA) algorithm to
compute reachability, and reports all functions referenced
by the SSA representation that were not found to be
reachable, grouped by package and sorted by position.

Also, a basic integration test.

Change-Id: Ide78b4e22d4f4066bf901e2d676e5058ca132827
Reviewed-on: https://go-review.googlesource.com/c/tools/+/507738
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Alan Donovan <adonovan@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>

internal/cmd/deadcode/deadcode.go

@@ -0,0 +1,240 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	_ "embed"
+	"flag"
+	"fmt"
+	"go/token"
+	"io"
+	"log"
+	"os"
+	"regexp"
+	"runtime"
+	"runtime/pprof"
+	"sort"
+	"strings"
+
+	"golang.org/x/tools/go/callgraph/rta"
+	"golang.org/x/tools/go/packages"
+	"golang.org/x/tools/go/ssa"
+	"golang.org/x/tools/go/ssa/ssautil"
+)
+
+//go:embed doc.go
+var doc string
+
+// flags
+var (
+	testFlag = flag.Bool("test", false, "include implicit test packages and executables")
+	tagsFlag = flag.String("tags", "", "comma-separated list of extra build tags (see: go help buildconstraint)")
+
+	filterFlag = flag.String("filter", "<module>", "report only packages matching this regular expression (default: module of first package)")
+	lineFlag   = flag.Bool("line", false, "show output in a line-oriented format")
+	cpuProfile = flag.String("cpuprofile", "", "write CPU profile to this file")
+	memProfile = flag.String("memprofile", "", "write memory profile to this file")
+)
+
+func usage() {
+	// Extract the content of the /* ... */ comment in doc.go.
+	_, after, _ := strings.Cut(doc, "/*\n")
+	doc, _, _ := strings.Cut(after, "*/")
+	io.WriteString(flag.CommandLine.Output(), doc+`
+Flags:
+
+`)
+	flag.PrintDefaults()
+}
+
+func main() {
+	log.SetPrefix("deadcode: ")
+	log.SetFlags(0) // no time prefix
+
+	flag.Usage = usage
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		usage()
+		os.Exit(2)
+	}
+
+	if *cpuProfile != "" {
+		f, err := os.Create(*cpuProfile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if err := pprof.StartCPUProfile(f); err != nil {
+			log.Fatal(err)
+		}
+		// NB: profile won't be written in case of error.
+		defer pprof.StopCPUProfile()
+	}
+
+	if *memProfile != "" {
+		f, err := os.Create(*memProfile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		// NB: profile won't be written in case of error.
+		defer func() {
+			runtime.GC() // get up-to-date statistics
+			if err := pprof.WriteHeapProfile(f); err != nil {
+				log.Fatalf("Writing memory profile: %v", err)
+			}
+			f.Close()
+		}()
+	}
+
+	// Load, parse, and type-check the complete program(s).
+	cfg := &packages.Config{
+		BuildFlags: []string{"-tags=" + *tagsFlag},
+		Mode:       packages.LoadAllSyntax | packages.NeedModule,
+		Tests:      *testFlag,
+	}
+	initial, err := packages.Load(cfg, flag.Args()...)
+	if err != nil {
+		log.Fatalf("Load: %v", err)
+	}
+	if len(initial) == 0 {
+		log.Fatalf("no packages")
+	}
+	if packages.PrintErrors(initial) > 0 {
+		log.Fatalf("packages contain errors")
+	}
+
+	// If -filter is unset, use first module (if available).
+	if *filterFlag == "<module>" {
+		if mod := initial[0].Module; mod != nil && mod.Path != "" {
+			*filterFlag = "^" + regexp.QuoteMeta(mod.Path) + "\\b"
+		} else {
+			*filterFlag = "" // match any
+		}
+	}
+	filter, err := regexp.Compile(*filterFlag)
+	if err != nil {
+		log.Fatalf("-filter: %v", err)
+	}
+
+	// Create SSA-form program representation
+	// and find main packages.
+	prog, pkgs := ssautil.AllPackages(initial, ssa.InstantiateGenerics)
+	prog.Build()
+
+	mains := ssautil.MainPackages(pkgs)
+	if len(mains) == 0 {
+		log.Fatalf("no main packages")
+	}
+	var roots []*ssa.Function
+	for _, main := range mains {
+		roots = append(roots, main.Func("init"), main.Func("main"))
+	}
+
+	// Compute the reachabilty from main.
+	// (We don't actually build a call graph.)
+	res := rta.Analyze(roots, false)
+
+	// Subtle: the -test flag causes us to analyze test variants
+	// such as "package p as compiled for p.test" or even "for q.test".
+	// This leads to multiple distinct ssa.Function instances that
+	// represent the same source declaration, and it is essentially
+	// impossible to discover this from the SSA representation
+	// (since it has lost the connection to go/packages.Package.ID).
+	//
+	// So, we de-duplicate such variants by position:
+	// if any one of them is live, we consider all of them live.
+	// (We use Position not Pos to avoid assuming that files common
+	// to packages "p" and "p [p.test]" were parsed only once.)
+	reachablePosn := make(map[token.Position]bool)
+	for fn := range res.Reachable {
+		if fn.Pos().IsValid() {
+			reachablePosn[prog.Fset.Position(fn.Pos())] = true
+		}
+	}
+
+	// Group unreachable functions by package path.
+	byPkgPath := make(map[string]map[*ssa.Function]bool)
+	for fn := range ssautil.AllFunctions(prog) {
+		if fn.Synthetic != "" {
+			continue // ignore synthetic wrappers etc
+		}
+
+		// Use generic, as instantiations may not have a Pkg.
+		if orig := fn.Origin(); orig != nil {
+			fn = orig
+		}
+
+		// Ignore unreachable nested functions.
+		// Literal functions passed as arguments to other
+		// functions are of course address-taken and there
+		// exists a dynamic call of that signature, so when
+		// they are unreachable, it is invariably because the
+		// parent is unreachable.
+		if fn.Parent() != nil {
+			continue
+		}
+
+		posn := prog.Fset.Position(fn.Pos())
+		if !reachablePosn[posn] {
+			reachablePosn[posn] = true // suppress dups with same pos
+
+			pkgpath := fn.Pkg.Pkg.Path()
+			m, ok := byPkgPath[pkgpath]
+			if !ok {
+				m = make(map[*ssa.Function]bool)
+				byPkgPath[pkgpath] = m
+			}
+			m[fn] = true
+		}
+	}
+
+	// Report dead functions grouped by packages.
+	// TODO(adonovan): use maps.Keys, twice.
+	pkgpaths := make([]string, 0, len(byPkgPath))
+	for pkgpath := range byPkgPath {
+		pkgpaths = append(pkgpaths, pkgpath)
+	}
+	sort.Strings(pkgpaths)
+	for _, pkgpath := range pkgpaths {
+		if !filter.MatchString(pkgpath) {
+			continue
+		}
+
+		m := byPkgPath[pkgpath]
+
+		// Print functions that appear within the same file in
+		// declaration order. This tends to keep related
+		// methods such as (T).Marshal and (*T).Unmarshal
+		// together better than sorting.
+		fns := make([]*ssa.Function, 0, len(m))
+		for fn := range m {
+			fns = append(fns, fn)
+		}
+		sort.Slice(fns, func(i, j int) bool {
+			xposn := prog.Fset.Position(fns[i].Pos())
+			yposn := prog.Fset.Position(fns[j].Pos())
+			if xposn.Filename != yposn.Filename {
+				return xposn.Filename < yposn.Filename
+			}
+			return xposn.Line < yposn.Line
+		})
+
+		// TODO(adonovan): add an option to skip (or indicate)
+		// dead functions in generated files (see ast.IsGenerated).
+
+		if *lineFlag {
+			// line-oriented output
+			for _, fn := range fns {
+				fmt.Println(fn)
+			}
+		} else {
+			// functions grouped by package
+			fmt.Printf("package %q\n", pkgpath)
+			for _, fn := range fns {
+				fmt.Printf("\tfunc %s\n", fn.RelString(fn.Pkg.Pkg))
+			}
+			fmt.Println()
+		}
+	}
+}

internal/cmd/deadcode/deadcode_test.go

@@ -0,0 +1,129 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main_test
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+
+	"golang.org/x/tools/internal/testenv"
+	"golang.org/x/tools/txtar"
+)
+
+// Test runs the deadcode command on each scenario
+// described by a testdata/*.txtar file.
+func Test(t *testing.T) {
+	testenv.NeedsTool(t, "go")
+	if runtime.GOOS == "android" {
+		t.Skipf("the dependencies are not available on android")
+	}
+
+	exe := buildDeadcode(t)
+
+	matches, err := filepath.Glob("testdata/*.txtar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, filename := range matches {
+		filename := filename
+		t.Run(filename, func(t *testing.T) {
+			t.Parallel()
+
+			ar, err := txtar.ParseFile(filename)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Parse archive comment as directives of these forms:
+			//
+			//    deadcode args...		command-line arguments
+			//  [!]want "quoted"		expected/unwanted string in output
+			//
+			var args []string
+			want := make(map[string]bool) // string -> sense
+			for _, line := range strings.Split(string(ar.Comment), "\n") {
+				line = strings.TrimSpace(line)
+				if line == "" || line[0] == '#' {
+					continue // skip blanks and comments
+				}
+
+				fields := strings.Fields(line)
+				switch kind := fields[0]; kind {
+				case "deadcode":
+					args = fields[1:] // lossy wrt spaces
+				case "want", "!want":
+					rest := line[len(kind):]
+					str, err := strconv.Unquote(strings.TrimSpace(rest))
+					if err != nil {
+						t.Fatalf("bad %s directive <<%s>>", kind, line)
+					}
+					want[str] = kind[0] != '!'
+				default:
+					t.Fatalf("%s: invalid directive %q", filename, kind)
+				}
+			}
+
+			// Write the archive files to the temp directory.
+			tmpdir := t.TempDir()
+			for _, f := range ar.Files {
+				filename := filepath.Join(tmpdir, f.Name)
+				if err := os.MkdirAll(filepath.Dir(filename), 0777); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(filename, f.Data, 0666); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			// Run the command.
+			cmd := exec.Command(exe, args...)
+			cmd.Stdout = new(bytes.Buffer)
+			cmd.Stderr = new(bytes.Buffer)
+			cmd.Dir = tmpdir
+			cmd.Env = append(os.Environ(), "GOPROXY=", "GO111MODULE=on")
+			if err := cmd.Run(); err != nil {
+				t.Fatalf("deadcode failed: %v (stderr=%s)", err, cmd.Stderr)
+			}
+
+			// Check each want directive.
+			got := fmt.Sprint(cmd.Stdout)
+			for str, sense := range want {
+				ok := true
+				if strings.Contains(got, str) != sense {
+					if sense {
+						t.Errorf("missing %q", str)
+					} else {
+						t.Errorf("unwanted %q", str)
+					}
+					ok = false
+				}
+				if !ok {
+					t.Errorf("got: <<%s>>", got)
+				}
+			}
+		})
+	}
+}
+
+// buildDeadcode builds the deadcode executable.
+// It returns its path, and a cleanup function.
+func buildDeadcode(t *testing.T) string {
+	bin := filepath.Join(t.TempDir(), "deadcode")
+	if runtime.GOOS == "windows" {
+		bin += ".exe"
+	}
+	cmd := exec.Command("go", "build", "-o", bin)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("Building deadcode: %v\n%s", err, out)
+	}
+	return bin
+}

internal/cmd/deadcode/doc.go

@@ -0,0 +1,58 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+The deadcode command reports unreachable functions in Go programs.
+
+Usage: deadcode [flags] package...
+
+The deadcode command loads a Go program from source then uses Rapid
+Type Analysis (RTA) to build a call graph of all the functions
+reachable from the program's main function. Any functions that are not
+reachable are reported as dead code, grouped by package.
+
+Packages are expressed in the notation of 'go list' (or other
+underlying build system if you are using an alternative
+golang.org/x/go/packages driver). Only executable (main) packages are
+considered starting points for the analysis.
+
+The -test flag causes it to analyze test executables too. Tests
+sometimes make use of functions that would otherwise appear to be dead
+code, and public API functions reported as dead with -test indicate
+possible gaps in your test coverage. Bear in mind that an Example test
+function without an "Output:" comment is merely documentation:
+it is dead code, and does not contribute coverage.
+
+The -filter flag restricts results to packages that match the provided
+regular expression; its default value is the module name of the first
+package. Use -filter= to display all results.
+
+Example: show all dead code within the gopls module:
+
+	$ deadcode -test golang.org/x/tools/gopls/...
+
+The analysis can soundly analyze dynamic calls though func values,
+interface methods, and reflection. However, it does not currently
+understand the aliasing created by //go:linkname directives, so it
+will fail to recognize that calls to a linkname-annotated function
+with no body in fact dispatch to the function named in the annotation.
+This may result in the latter function being spuriously reported as dead.
+
+In any case, just because a function is reported as dead does not mean
+it is unconditionally safe to delete it. For example, a dead function
+may be referenced (by another dead function), and a dead method may be
+required to satisfy an interface (that is never called).
+Some judgement is required.
+
+The analysis is valid only for a single GOOS/GOARCH/-tags configuration,
+so a function reported as dead may be live in a different configuration.
+Consider running the tool once for each configuration of interest.
+Use the -line flag to emit a line-oriented output that makes it
+easier to compute the intersection of results across all runs.
+
+THIS TOOL IS EXPERIMENTAL and its interface may change.
+At some point it may be published at cmd/deadcode.
+In the meantime, please give us feedback at github.com/golang/go/issues.
+*/
+package main

internal/cmd/deadcode/testdata/basic.txtar

@@ -0,0 +1,32 @@
+# Test of basic functionality.
+
+ deadcode -filter= example.com
+
+ want "func (T).Goodbye"
+!want "func (T).Hello"
+ want "func unreferenced"
+
+ want "func Scanf"
+ want "func Printf"
+!want "func Println"
+
+-- go.mod --
+module example.com
+go 1.18
+
+-- main.go --
+package main
+
+import "fmt"
+
+type T int
+
+func main() {
+	var x T
+	x.Hello()
+}
+
+func (T) Hello() { fmt.Println("hello") }
+func (T) Goodbye() { fmt.Println("goodbye") }
+
+func unreferenced() {}

internal/cmd/deadcode/testdata/filterflag.txtar

@@ -0,0 +1,39 @@
+# Test of -filter flag.
+
+ deadcode -filter=other.net example.com
+
+ want `package "other.net"`
+ want `func Dead`
+!want `func Live`
+
+!want `package "example.com"`
+!want `func unreferenced`
+
+-- go.work --
+use example.com
+use other.net
+
+-- example.com/go.mod --
+module example.com
+go 1.18
+
+-- example.com/main.go --
+package main
+
+import "other.net"
+
+func main() {
+	other.Live()
+}
+
+func unreferenced() {}
+
+-- other.net/go.mod --
+module other.net
+go 1.18
+
+-- other.net/other.go --
+package other
+
+func Live() {}
+func Dead() {}

internal/cmd/deadcode/testdata/lineflag.txtar

@@ -0,0 +1,32 @@
+# Test of -line output.
+
+ deadcode -line -filter= example.com
+
+ want "(example.com.T).Goodbye"
+!want "(example.com.T).Hello"
+ want "example.com.unreferenced"
+
+ want "fmt.Scanf"
+ want "fmt.Printf"
+!want "fmt.Println"
+
+-- go.mod --
+module example.com
+go 1.18
+
+-- main.go --
+package main
+
+import "fmt"
+
+type T int
+
+func main() {
+	var x T
+	x.Hello()
+}
+
+func (T) Hello() { fmt.Println("hello") }
+func (T) Goodbye() { fmt.Println("goodbye") }
+
+func unreferenced() {}

internal/cmd/deadcode/testdata/testflag.txtar

@@ -0,0 +1,42 @@
+# Test of -test flag.
+
+deadcode -test -filter=example.com example.com/p
+
+ want "func Dead"
+!want "func Live1"
+!want "func Live2"
+
+ want "func ExampleDead"
+!want "func ExampleLive"
+
+-- go.mod --
+module example.com
+go 1.18
+
+-- p/p.go --
+package p
+
+func Live1() {}
+func Live2() {}
+func Dead() {}
+
+-- p/p_test.go --
+package p_test
+
+import "example.com/p"
+
+import "testing"
+
+func Test(t *testing.T) {
+	p.Live1()
+}
+
+func ExampleLive() {
+	p.Live2()
+	// Output:
+}
+
+// A test Example function without an "Output:" comment is never executed.
+func ExampleDead() {
+	p.Dead()
+}