go
/
tools
зеркало из https://github.com/golang/tools.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

gopls/internal/telemetry/cmd/stacks: generate CodeSearch links 

This CL causes the stacks command to mark up each stack as a
set of links to CodeSearch. In order to do that, it needs to
build the gopls executable at the correct version of gopls and Go
and for the right GOOS and GOARCH, read the pclntab out of the
executable (which is the only authority on how to decode the
symbol names that appear in the stack counter), and then
construct CodeSearch URLs from (version, file, line) triples.

The expensive steps are cached in /tmp/gopls-stacks so that they
are paid infrequently in a typical stacks run.

See https://github.com/golang/go/issues/67288 for an example
of the updated output.

Fixes golang/go#64654

Change-Id: If1c3e42af5550114515b47a22dfa036e8da27143
Reviewed-on: https://go-review.googlesource.com/c/tools/+/611840
Auto-Submit: Alan Donovan <adonovan@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Robert Findley <rfindley@google.com>
This commit is contained in:
Alan Donovan 2024-09-10 21:26:56 -04:00 коммит произвёл Gopher Robot
Родитель beed481fb5
Коммит 3e49191340
1 изменённых файлов: 252 добавлений и 21 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

273
gopls/internal/telemetry/cmd/stacks/stacks.go
Просмотреть файл

@ -7,8 +7,12 @@
// issue, creating new issues as needed.
package main
// TODO(adonovan): create a proper package with tests. Much of this
// machinery might find wider use in other x/telemetry clients.
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"flag"
@ -19,8 +23,10 @@ import (
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
@ -68,8 +74,8 @@ func main() {
token = string(bytes.TrimSpace(content))
}
// Maps stack text to Version/GoVersion/GOOS/GOARCH string to counter.
stacks := make(map[string]map[string]int64)
// Maps stack text to Info to count.
stacks := make(map[string]map[Info]int64)
var distinctStacks int
// Maps stack to a telemetry URL.
@ -112,7 +118,7 @@ func main() {
}
sort.Strings(clients)
if len(clients) > 0 {
clientSuffix = " " + strings.Join(clients, ",")
clientSuffix = strings.Join(clients, ",")
}
// Ignore @devel versions as they correspond to
@ -124,14 +130,18 @@ func main() {
distinctStacks++
info := fmt.Sprintf("%s@%s %s %s/%s%s",
prog.Program, prog.Version,
prog.GoVersion, prog.GOOS, prog.GOARCH,
clientSuffix)
info := Info{
Program: prog.Program,
Version: prog.Version,
GoVersion: prog.GoVersion,
GOOS: prog.GOOS,
GOARCH: prog.GOARCH,
Client: clientSuffix,
}
for stack, count := range prog.Stacks {
counts := stacks[stack]
if counts == nil {
counts = make(map[string]int64)
counts = make(map[Info]int64)
stacks[stack] = counts
}
counts[info] += count
@ -176,8 +186,6 @@ func main() {
}
}
fmt.Printf("Found %d distinct stacks in last %v days:\n", distinctStacks, *daysFlag)
// For each stack, show existing issue or create a new one.
// Aggregate stack IDs by issue summary.
var (
@ -188,23 +196,28 @@ func main() {
for stack, counts := range stacks {
id := stackID(stack)
var info0 Info // an arbitrary Info for this stack
var total int64
for _, count := range counts {
for info, count := range counts {
info0 = info
total += count
}
if issue, ok := issuesByStackID[id]; ok {
// existing issue
// TODO(adonovan): use ANSI tty color codes for Issue.State.
summary := fmt.Sprintf("#%d: %s [%s]",
issue.Number, issue.Title, issue.State)
existingIssues[summary] += total
} else {
// new issue
title := newIssue(stack, id, stackToURL[stack], counts)
title := newIssue(stack, id, info0, stackToURL[stack], counts)
summary := fmt.Sprintf("%s: %s [%s]", id, title, "new")
newIssues[summary] += total
}
}
fmt.Printf("Found %d distinct stacks in last %v days:\n", distinctStacks, *daysFlag)
print := func(caption string, issues map[string]int64) {
// Print items in descending frequency.
keys := moremaps.KeySlice(issues)
@ -214,6 +227,7 @@ func main() {
fmt.Printf("%s issues:\n", caption)
for _, summary := range keys {
count := issues[summary]
// TODO(adonovan): use ANSI tty codes to show high n values in bold.
fmt.Printf("%s (n=%d)\n", summary, count)
}
}
@ -221,12 +235,26 @@ func main() {
print("New", newIssues)
}
type Info struct {
Program string // "golang.org/x/tools/gopls"
Version, GoVersion string // e.g. "gopls/v0.16.1", "go1.23"
GOOS, GOARCH string
Client string // e.g. "vscode"
}
func (info Info) String() string {
return fmt.Sprintf("%s@%s %s %s/%s %s",
info.Program, info.Version,
info.GoVersion, info.GOOS, info.GOARCH,
info.Client)
}
// stackID returns a 32-bit identifier for a stack
// suitable for use in GitHub issue titles.
func stackID(stack string) string {
// Encode it using base64 (6 bytes) for brevity,
// as a single issue's body might contain multiple IDs
// if separate issues with same cause wre manually de-duped,
// if separate issues with same cause were manually de-duped,
// e.g. "AAAAAA, BBBBBB"
//
// https://hbfs.wordpress.com/2012/03/30/finding-collisions:
@ -245,11 +273,14 @@ func stackID(stack string) string {
// manually de-dup the issue before deciding whether to submit the form.)
//
// It returns the title.
func newIssue(stack, id, jsonURL string, counts map[string]int64) string {
func newIssue(stack, id string, info Info, jsonURL string, counts map[Info]int64) string {
// Use a heuristic to find a suitable symbol to blame
// in the title: the first public function or method
// of a public type, in gopls, to appear in the stack
// trace. We can always refine it later.
//
// TODO(adonovan): include in the issue a source snippet ±5
// lines around the PC in this symbol.
var symbol string
for _, line := range strings.Split(stack, "\n") {
// Look for:
@ -273,19 +304,92 @@ func newIssue(stack, id, jsonURL string, counts map[string]int64) string {
}
// Populate the form (title, body, label)
title := fmt.Sprintf("x/tools/gopls:%s bug reported by telemetry", symbol)
title := fmt.Sprintf("x/tools/gopls: bug in %s", symbol)
body := new(bytes.Buffer)
fmt.Fprintf(body, "This stack `%s` was [reported by telemetry](%s):\n\n",
id, jsonURL)
fmt.Fprintf(body, "```\n%s\n```\n", stack)
// Read the mapping from symbols to file/line.
pclntab, err := readPCLineTable(info)
if err != nil {
log.Fatal(err)
}
// Parse the stack and get the symbol names out.
for _, line := range strings.Split(stack, "\n") {
// e.g. "golang.org/x/tools/gopls/foo.(*Type).Method.inlined.func3:+5"
symbol, offset, ok := strings.Cut(line, ":")
if !ok {
// Not a symbol (perhaps stack counter title: "gopls/bug"?)
fmt.Fprintf(body, "`%s`\n", line)
continue
}
fileline, ok := pclntab[symbol]
if !ok {
// objdump reports ELF symbol names, which in
// rare cases may be the Go symbols of
// runtime.CallersFrames mangled by (e.g.) the
// addition of .abi0 suffix; see
// https://github.com/golang/go/issues/69390#issuecomment-2343795920
// So this should not be a hard error.
log.Printf("no pclntab info for symbol: %s", symbol)
fmt.Fprintf(body, "`%s`\n", line)
continue
}
if offset == "" {
log.Fatalf("missing line offset: %s", line)
}
offsetNum, err := strconv.Atoi(offset[1:])
if err != nil {
log.Fatalf("invalid line offset: %s", line)
}
linenum := fileline.line
switch offset[0] {
case '-':
linenum -= offsetNum
case '+':
linenum += offsetNum
case '=':
linenum = offsetNum
}
// Construct CodeSearch URL.
var url string
if firstSegment, _, _ := strings.Cut(fileline.file, "/"); !strings.Contains(firstSegment, ".") {
// std
// (First segment is a dir beneath GOROOT/src, not a module domain name.)
url = fmt.Sprintf("https://cs.opensource.google/go/go/+/%s:src/%s;l=%d",
info.GoVersion, fileline.file, linenum)
} else if rest, ok := strings.CutPrefix(fileline.file, "golang.org/x/tools"); ok {
// in x/tools repo (tools or gopls module)
if rest[0] == '/' {
// "golang.org/x/tools/gopls" -> "gopls"
rest = rest[1:]
} else if rest[0] == '@' {
// "golang.org/x/tools@version/dir/file.go" -> "dir/file.go"
rest = rest[strings.Index(rest, "/")+1:]
}
url = fmt.Sprintf("https://cs.opensource.google/go/x/tools/+/%s:%s;l=%d",
"gopls/"+info.Version, rest, linenum)
} else {
// TODO(adonovan): support other module dependencies of gopls.
log.Printf("no CodeSearch URL for %q (%s:%d)",
symbol, fileline.file, linenum)
}
// Emit stack frame.
if url != "" {
fmt.Fprintf(body, "- [`%s`](%s)\n", line, url)
} else {
fmt.Fprintf(body, "- `%s`\n", line)
}
}
// Add counts, gopls version, and platform info.
// This isn't very precise but should provide clues.
//
// TODO(adonovan): link each stack (ideally each frame) to source:
// https://cs.opensource.google/go/x/tools/+/gopls/VERSION:gopls/FILE;l=LINE
// (Requires parsing stack, shallow-cloning gopls module at that tag, and
// computing correct line offsets. Would be labor-saving though.)
fmt.Fprintf(body, "```\n")
for info, count := range counts {
fmt.Fprintf(body, "%s (%d)\n", info, count)
@ -371,3 +475,130 @@ func min(x, y int) int {
return y
}
}
// -- pclntab --
type FileLine struct {
file string // "module@version/dir/file.go" or path relative to $GOROOT/src
line int
}
// readPCLineTable builds the gopls executable specified by info,
// reads its PC-to-line-number table, and returns the file/line of
// each TEXT symbol.
func readPCLineTable(info Info) (map[string]FileLine, error) {
// The stacks dir will be a semi-durable temp directory
// (i.e. lasts for at least hours) holding source trees
// and executables we have build recently.
//
// Each subdir will hold a specific revision.
stacksDir := "/tmp/gopls-stacks"
if err := os.MkdirAll(stacksDir, 0777); err != nil {
return nil, fmt.Errorf("can't create stacks dir: %v", err)
}
// Fetch the source for the tools repo,
// shallow-cloning just the desired revision.
// (Skip if it's already cloned.)
revDir := filepath.Join(stacksDir, info.Version)
if !fileExists(revDir) {
log.Printf("cloning tools@gopls/%s", info.Version)
if err := shallowClone(revDir, "https://go.googlesource.com/tools", "gopls/"+info.Version); err != nil {
return nil, fmt.Errorf("clone: %v", err)
}
}
// Build the executable with the correct GOTOOLCHAIN, GOOS, GOARCH.
// Use -trimpath for normalized file names.
// (Skip if it's already built.)
exe := fmt.Sprintf("exe-%s.%s-%s", info.GoVersion, info.GOOS, info.GOARCH)
cmd := exec.Command("go", "build", "-trimpath", "-o", "../"+exe)
cmd.Dir = filepath.Join(revDir, "gopls")
cmd.Env = append(os.Environ(),
"GOTOOLCHAIN="+info.GoVersion,
"GOOS="+info.GOOS,
"GOARCH="+info.GOARCH,
)
if !fileExists(filepath.Join(revDir, exe)) {
log.Printf("building %s@%s with %s on %s/%s",
info.Program, info.Version, info.GoVersion, info.GOOS, info.GOARCH)
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("building: %v", err)
}
}
// Read pclntab of executable.
cmd = exec.Command("go", "tool", "objdump", exe)
cmd.Stdout = new(strings.Builder)
cmd.Stderr = os.Stderr
cmd.Dir = revDir
cmd.Env = append(os.Environ(),
"GOTOOLCHAIN="+info.GoVersion,
"GOOS="+info.GOOS,
"GOARCH="+info.GOARCH,
)
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("reading pclntab %v", err)
}
pclntab := make(map[string]FileLine)
lines := strings.Split(fmt.Sprint(cmd.Stdout), "\n")
for i, line := range lines {
// Each function is of this form:
//
// TEXT symbol(SB) filename
// basename.go:line instruction
// ...
if !strings.HasPrefix(line, "TEXT ") {
continue
}
fields := strings.Fields(line)
if len(fields) != 3 {
continue // symbol without file (e.g. go:buildid)
}
symbol := strings.TrimSuffix(fields[1], "(SB)")
filename := fields[2]
_, line, ok := strings.Cut(strings.Fields(lines[i+1])[0], ":")
if !ok {
return nil, fmt.Errorf("can't parse 'basename.go:line' from first instruction of %s:\n%s",
symbol, line)
}
linenum, err := strconv.Atoi(line)
if err != nil {
return nil, fmt.Errorf("can't parse line number of %s: %s", symbol, line)
}
pclntab[symbol] = FileLine{filename, linenum}
}
return pclntab, nil
}
// shallowClone performs a shallow clone of repo into dir at the given
// 'commitish' ref (any commit reference understood by git).
//
// The directory dir must not already exist.
func shallowClone(dir, repo, commitish string) error {
if err := os.Mkdir(dir, 0750); err != nil {
return fmt.Errorf("creating dir for %s: %v", repo, err)
}
// Set a timeout for git fetch. If this proves flaky, it can be removed.
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
// Use a shallow fetch to download just the relevant commit.
shInit := fmt.Sprintf("git init && git fetch --depth=1 %q %q && git checkout FETCH_HEAD", repo, commitish)
initCmd := exec.CommandContext(ctx, "/bin/sh", "-c", shInit)
initCmd.Dir = dir
if output, err := initCmd.CombinedOutput(); err != nil {
return fmt.Errorf("checking out %s: %v\n%s", repo, err, output)
}
return nil
}
func fileExists(filename string) bool {
_, err := os.Stat(filename)
return err == nil
}