diff --git a/cmd/benchcmp/benchcmp.go b/cmd/benchcmp/benchcmp.go new file mode 100644 index 000000000..26ffc345a --- /dev/null +++ b/cmd/benchcmp/benchcmp.go @@ -0,0 +1,150 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Benchcmp is a utility for comparing benchmark runs. +package main + +import ( + "flag" + "fmt" + "os" + "sort" + "strconv" + "text/tabwriter" +) + +var ( + changedOnly = flag.Bool("changed", false, "show only benchmarks that have changed") +) + +const usageFooter = ` +Each input file should be from: + go test -test.run=NONE -test.bench=. > [old,new].txt + +Benchcmp compares old and new for each benchmark. + +If -test.benchmem=true is added to the "go test" command +benchcmp will also compare memory allocations. +` + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: %s old.txt new.txt\n\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprint(os.Stderr, usageFooter) + os.Exit(2) + } + flag.Parse() + if flag.NArg() != 2 { + flag.Usage() + } + + before := parseFile(flag.Arg(0)) + after := parseFile(flag.Arg(1)) + + cmps, warnings := Correlate(before, after) + + for _, warn := range warnings { + fmt.Fprintln(os.Stderr, warn) + } + + if len(cmps) == 0 { + fatal("benchcmp: no repeated benchmarks") + } + + w := new(tabwriter.Writer) + w.Init(os.Stdout, 0, 0, 5, ' ', 0) + defer w.Flush() + + var header bool // Has the header has been displayed yet for a given block? + + sort.Sort(ByDeltaNsOp(cmps)) + for _, cmp := range cmps { + if !cmp.Measured(NsOp) { + continue + } + if delta := cmp.DeltaNsOp(); !*changedOnly || delta.Changed() { + if !header { + fmt.Fprintf(w, "benchmark\told ns/op\tnew ns/op\tdelta\t\n") + header = true + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t\n", cmp.Name(), formatNs(cmp.Before.NsOp), formatNs(cmp.After.NsOp), delta.Percent()) + } + } + + header = false + sort.Sort(ByDeltaMbS(cmps)) + for _, cmp := range cmps { + if !cmp.Measured(MbS) { + continue + } + if delta := cmp.DeltaMbS(); !*changedOnly || delta.Changed() { + if !header { + fmt.Fprintf(w, "\nbenchmark\told MB/s\tnew MB/s\tspeedup\t\n") + header = true + } + fmt.Fprintf(w, "%s\t%.2f\t%.2f\t%s\t\n", cmp.Name(), cmp.Before.MbS, cmp.After.MbS, delta.Multiple()) + } + } + + header = false + sort.Sort(ByDeltaAllocsOp(cmps)) + for _, cmp := range cmps { + if !cmp.Measured(AllocsOp) { + continue + } + if delta := cmp.DeltaAllocsOp(); !*changedOnly || delta.Changed() { + if !header { + fmt.Fprintf(w, "\nbenchmark\told allocs\tnew allocs\tdelta\t\n") + header = true + } + fmt.Fprintf(w, "%s\t%d\t%d\t%s\t\n", cmp.Name(), cmp.Before.AllocsOp, cmp.After.AllocsOp, delta.Percent()) + } + } + + header = false + sort.Sort(ByDeltaBOp(cmps)) + for _, cmp := range cmps { + if !cmp.Measured(BOp) { + continue + } + if delta := cmp.DeltaBOp(); !*changedOnly || delta.Changed() { + if !header { + fmt.Fprintf(w, "\nbenchmark\told bytes\tnew bytes\tdelta\t\n") + header = true + } + fmt.Fprintf(w, "%s\t%d\t%d\t%s\t\n", cmp.Name(), cmp.Before.BOp, cmp.After.BOp, cmp.DeltaBOp().Percent()) + } + } +} + +func fatal(msg interface{}) { + fmt.Fprintln(os.Stderr, msg) + os.Exit(1) +} + +func parseFile(path string) BenchSet { + f, err := os.Open(path) + if err != nil { + fatal(err) + } + bb, err := ParseBenchSet(f) + if err != nil { + fatal(err) + } + return bb +} + +// formatNs formats ns measurements to expose a useful amount of +// precision. It mirrors the ns precision logic of testing.B. +func formatNs(ns float64) string { + prec := 0 + switch { + case ns < 10: + prec = 2 + case ns < 100: + prec = 1 + } + return strconv.FormatFloat(ns, 'f', prec, 64) +} diff --git a/cmd/benchcmp/compare.go b/cmd/benchcmp/compare.go new file mode 100644 index 000000000..c4f417647 --- /dev/null +++ b/cmd/benchcmp/compare.go @@ -0,0 +1,140 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "math" +) + +// BenchCmp is a pair of benchmarks. +type BenchCmp struct { + Before *Bench + After *Bench +} + +// Correlate correlates benchmarks from two BenchSets. +func Correlate(before, after BenchSet) (cmps []BenchCmp, warnings []string) { + cmps = make([]BenchCmp, 0, len(after)) + for name, beforebb := range before { + afterbb := after[name] + if len(beforebb) != len(afterbb) { + warnings = append(warnings, fmt.Sprintf("ignoring %s: before has %d instances, after has %d", name, len(beforebb), len(afterbb))) + continue + } + for i, beforeb := range beforebb { + afterb := afterbb[i] + cmps = append(cmps, BenchCmp{beforeb, afterb}) + } + } + return +} + +func (c BenchCmp) Name() string { return c.Before.Name } +func (c BenchCmp) String() string { return fmt.Sprintf("<%s, %s>", c.Before, c.After) } +func (c BenchCmp) Measured(flag int) bool { return c.Before.Measured&c.After.Measured&flag != 0 } +func (c BenchCmp) DeltaNsOp() Delta { return Delta{c.Before.NsOp, c.After.NsOp} } +func (c BenchCmp) DeltaMbS() Delta { return Delta{c.Before.MbS, c.After.MbS} } +func (c BenchCmp) DeltaBOp() Delta { return Delta{float64(c.Before.BOp), float64(c.After.BOp)} } +func (c BenchCmp) DeltaAllocsOp() Delta { + return Delta{float64(c.Before.AllocsOp), float64(c.After.AllocsOp)} +} + +// Delta is the before and after value for a benchmark measurement. +// Both must be non-negative. +type Delta struct { + Before float64 + After float64 +} + +// mag calculates the magnitude of a change, regardless of the direction of +// the change. mag is intended for sorting and has no independent meaning. +func (d Delta) mag() float64 { + switch { + case d.Before != 0 && d.After != 0 && d.Before >= d.After: + return d.After / d.Before + case d.Before != 0 && d.After != 0 && d.Before < d.After: + return d.Before / d.After + case d.Before == 0 && d.After == 0: + return 1 + default: + // 0 -> 1 or 1 -> 0 + // These are significant changes and worth surfacing. + return math.Inf(1) + } +} + +// Changed reports whether the benchmark quantities are different. +func (d Delta) Changed() bool { return d.Before != d.After } + +// Float64 returns After / Before. If Before is 0, Float64 returns +// 1 if After is also 0, and +Inf otherwise. +func (d Delta) Float64() float64 { + switch { + case d.Before != 0: + return d.After / d.Before + case d.After == 0: + return 1 + default: + return math.Inf(1) + } +} + +// Percent formats a Delta as a percent change, ranging from -100% up. +func (d Delta) Percent() string { + return fmt.Sprintf("%+.2f%%", 100*d.Float64()-100) +} + +// Multiple formats a Delta as a multiplier, ranging from 0.00x up. +func (d Delta) Multiple() string { + return fmt.Sprintf("%.2fx", d.Float64()) +} + +func (d Delta) String() string { + return fmt.Sprintf("Δ(%f, %f)", d.Before, d.After) +} + +// lessByDelta provides lexicographic ordering: +// * largest delta by magnitude +// * alphabetic by name +func lessByDelta(i, j BenchCmp, calcDelta func(BenchCmp) Delta) bool { + iDelta, jDelta := calcDelta(i).mag(), calcDelta(j).mag() + if iDelta != jDelta { + return iDelta < jDelta + } + return i.Name() < j.Name() +} + +// ByDeltaNsOp sorts BenchCmps lexicographically by change +// in ns/op, descending, then by benchmark name. +type ByDeltaNsOp []BenchCmp + +func (x ByDeltaNsOp) Len() int { return len(x) } +func (x ByDeltaNsOp) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x ByDeltaNsOp) Less(i, j int) bool { return lessByDelta(x[i], x[j], BenchCmp.DeltaNsOp) } + +// ByDeltaMbS sorts BenchCmps lexicographically by change +// in MB/s, descending, then by benchmark name. +type ByDeltaMbS []BenchCmp + +func (x ByDeltaMbS) Len() int { return len(x) } +func (x ByDeltaMbS) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x ByDeltaMbS) Less(i, j int) bool { return lessByDelta(x[i], x[j], BenchCmp.DeltaMbS) } + +// ByDeltaBOp sorts BenchCmps lexicographically by change +// in B/op, descending, then by benchmark name. +type ByDeltaBOp []BenchCmp + +func (x ByDeltaBOp) Len() int { return len(x) } +func (x ByDeltaBOp) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x ByDeltaBOp) Less(i, j int) bool { return lessByDelta(x[i], x[j], BenchCmp.DeltaBOp) } + +// ByDeltaAllocsOp sorts BenchCmps lexicographically by change +// in allocs/op, descending, then by benchmark name. +type ByDeltaAllocsOp []BenchCmp + +func (x ByDeltaAllocsOp) Len() int { return len(x) } +func (x ByDeltaAllocsOp) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x ByDeltaAllocsOp) Less(i, j int) bool { return lessByDelta(x[i], x[j], BenchCmp.DeltaAllocsOp) } diff --git a/cmd/benchcmp/compare_test.go b/cmd/benchcmp/compare_test.go new file mode 100644 index 000000000..1513f58f3 --- /dev/null +++ b/cmd/benchcmp/compare_test.go @@ -0,0 +1,124 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "math" + "reflect" + "sort" + "testing" +) + +func TestDelta(t *testing.T) { + cases := []struct { + before float64 + after float64 + mag float64 + f float64 + changed bool + pct string + mult string + }{ + {before: 1, after: 1, mag: 1, f: 1, changed: false, pct: "+0.00%", mult: "1.00x"}, + {before: 1, after: 2, mag: 0.5, f: 2, changed: true, pct: "+100.00%", mult: "2.00x"}, + {before: 2, after: 1, mag: 0.5, f: 0.5, changed: true, pct: "-50.00%", mult: "0.50x"}, + {before: 0, after: 0, mag: 1, f: 1, changed: false, pct: "+0.00%", mult: "1.00x"}, + {before: 1, after: 0, mag: math.Inf(1), f: 0, changed: true, pct: "-100.00%", mult: "0.00x"}, + {before: 0, after: 1, mag: math.Inf(1), f: math.Inf(1), changed: true, pct: "+Inf%", mult: "+Infx"}, + } + for _, tt := range cases { + d := Delta{tt.before, tt.after} + if want, have := tt.mag, d.mag(); want != have { + t.Errorf("%s.mag(): want %f have %f", d, want, have) + } + if want, have := tt.f, d.Float64(); want != have { + t.Errorf("%s.Float64(): want %f have %f", d, want, have) + } + if want, have := tt.changed, d.Changed(); want != have { + t.Errorf("%s.Changed(): want %t have %t", d, want, have) + } + if want, have := tt.pct, d.Percent(); want != have { + t.Errorf("%s.Percent(): want %q have %q", d, want, have) + } + if want, have := tt.mult, d.Multiple(); want != have { + t.Errorf("%s.Multiple(): want %q have %q", d, want, have) + } + } +} + +func TestCorrelate(t *testing.T) { + // Benches that are going to be successfully correlated get N thus: + // 0x + // Read this: " of , from ". + before := BenchSet{ + "BenchmarkOneEach": []*Bench{{Name: "BenchmarkOneEach", N: 0x11b}}, + "BenchmarkOneToNone": []*Bench{{Name: "BenchmarkOneToNone"}}, + "BenchmarkOneToTwo": []*Bench{{Name: "BenchmarkOneToTwo"}}, + "BenchmarkTwoToOne": []*Bench{ + {Name: "BenchmarkTwoToOne"}, + {Name: "BenchmarkTwoToOne"}, + }, + "BenchmarkTwoEach": []*Bench{ + {Name: "BenchmarkTwoEach", N: 0x12b}, + {Name: "BenchmarkTwoEach", N: 0x22b}, + }, + } + + after := BenchSet{ + "BenchmarkOneEach": []*Bench{{Name: "BenchmarkOneEach", N: 0x11a}}, + "BenchmarkNoneToOne": []*Bench{{Name: "BenchmarkNoneToOne"}}, + "BenchmarkTwoToOne": []*Bench{{Name: "BenchmarkTwoToOne"}}, + "BenchmarkOneToTwo": []*Bench{ + {Name: "BenchmarkOneToTwo"}, + {Name: "BenchmarkOneToTwo"}, + }, + "BenchmarkTwoEach": []*Bench{ + {Name: "BenchmarkTwoEach", N: 0x12a}, + {Name: "BenchmarkTwoEach", N: 0x22a}, + }, + } + + pairs, errs := Correlate(before, after) + + // Fail to match: BenchmarkOneToNone, BenchmarkOneToTwo, BenchmarkTwoToOne. + // Correlate does not notice BenchmarkNoneToOne. + if len(errs) != 3 { + t.Errorf("Correlated expected 4 errors, got %d: %v", len(errs), errs) + } + + // Want three correlated pairs: one BenchmarkOneEach, two BenchmarkTwoEach. + if len(pairs) != 3 { + t.Fatalf("Correlated expected 3 pairs, got %v", pairs) + } + + for _, pair := range pairs { + if pair.Before.N&0xF != 0xb { + t.Errorf("unexpected Before in pair %s", pair) + } + if pair.After.N&0xF != 0xa { + t.Errorf("unexpected After in pair %s", pair) + } + if pair.Before.N>>4 != pair.After.N>>4 { + t.Errorf("mismatched pair %s", pair) + } + } +} + +func TestBenchCmpSorting(t *testing.T) { + // Test just one sort order; they are symmetric. + c := []BenchCmp{ + {&Bench{Name: "BenchmarkMuchFaster", NsOp: 10}, &Bench{Name: "BenchmarkMuchFaster", NsOp: 1}}, + {&Bench{Name: "BenchmarkSameB", NsOp: 5}, &Bench{Name: "BenchmarkSameB", NsOp: 5}}, + {&Bench{Name: "BenchmarkSameA", NsOp: 5}, &Bench{Name: "BenchmarkSameA", NsOp: 5}}, + {&Bench{Name: "BenchmarkSlower", NsOp: 10}, &Bench{Name: "BenchmarkSlower", NsOp: 11}}, + } + + sort.Sort(ByDeltaNsOp(c)) + want := []string{"BenchmarkMuchFaster", "BenchmarkSlower", "BenchmarkSameA", "BenchmarkSameB"} + have := []string{c[0].Name(), c[1].Name(), c[2].Name(), c[3].Name()} + if !reflect.DeepEqual(want, have) { + t.Errorf("ByDeltaNsOp incorrect sorting: want %v have %v", want, have) + } +} diff --git a/cmd/benchcmp/parse.go b/cmd/benchcmp/parse.go new file mode 100644 index 000000000..5df7dfe00 --- /dev/null +++ b/cmd/benchcmp/parse.go @@ -0,0 +1,123 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" +) + +// Flags used by Bench.Measured to indicate +// which measurements a Bench contains. +const ( + NsOp = 1 << iota + MbS + BOp + AllocsOp +) + +// Bench is one run of a single benchmark. +type Bench struct { + Name string // benchmark name + N int // number of iterations + NsOp float64 // nanoseconds per iteration + MbS float64 // MB processed per second + BOp uint64 // bytes allocated per iteration + AllocsOp uint64 // allocs per iteration + Measured int // which measurements were recorded +} + +// ParseLine extracts a Bench from a single line of testing.B output. +func ParseLine(line string) (*Bench, error) { + fields := strings.Fields(line) + + // Two required, positional fields: Name and iterations. + if len(fields) < 2 { + return nil, fmt.Errorf("two fields required, have %d", len(fields)) + } + if !strings.HasPrefix(fields[0], "Benchmark") { + return nil, fmt.Errorf(`first field does not start with "Benchmark`) + } + n, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err + } + b := &Bench{Name: fields[0], N: n} + + // Parse any remaining pairs of fields; we've parsed one pair already. + for i := 1; i < len(fields)/2; i++ { + b.parseMeasurement(fields[i*2], fields[i*2+1]) + } + return b, nil +} + +func (b *Bench) parseMeasurement(quant string, unit string) { + switch unit { + case "ns/op": + if f, err := strconv.ParseFloat(quant, 64); err == nil { + b.NsOp = f + b.Measured |= NsOp + } + case "MB/s": + if f, err := strconv.ParseFloat(quant, 64); err == nil { + b.MbS = f + b.Measured |= MbS + } + case "B/op": + if i, err := strconv.ParseUint(quant, 10, 64); err == nil { + b.BOp = i + b.Measured |= BOp + } + case "allocs/op": + if i, err := strconv.ParseUint(quant, 10, 64); err == nil { + b.AllocsOp = i + b.Measured |= AllocsOp + } + } +} + +func (b *Bench) String() string { + buf := new(bytes.Buffer) + fmt.Fprintf(buf, "%s %d", b.Name, b.N) + if b.Measured&NsOp != 0 { + fmt.Fprintf(buf, " %.2f ns/op", b.NsOp) + } + if b.Measured&MbS != 0 { + fmt.Fprintf(buf, " %.2f MB/s", b.MbS) + } + if b.Measured&BOp != 0 { + fmt.Fprintf(buf, " %d B/op", b.BOp) + } + if b.Measured&AllocsOp != 0 { + fmt.Fprintf(buf, " %d allocs/op", b.AllocsOp) + } + return buf.String() +} + +// BenchSet is a collection of benchmarks from one +// testing.B run, keyed by name to faciliate comparison. +type BenchSet map[string][]*Bench + +// Parse extracts a BenchSet from testing.B output. Parse +// preserves the order of benchmarks that have identical names. +func ParseBenchSet(r io.Reader) (BenchSet, error) { + bb := make(BenchSet) + scan := bufio.NewScanner(r) + for scan.Scan() { + if b, err := ParseLine(scan.Text()); err == nil { + bb[b.Name] = append(bb[b.Name], b) + } + } + + if err := scan.Err(); err != nil { + return nil, err + } + + return bb, nil +} diff --git a/cmd/benchcmp/parse_test.go b/cmd/benchcmp/parse_test.go new file mode 100644 index 000000000..04148633f --- /dev/null +++ b/cmd/benchcmp/parse_test.go @@ -0,0 +1,150 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseLine(t *testing.T) { + cases := []struct { + line string + want *Bench + err bool // expect an error + }{ + { + line: "BenchmarkEncrypt 100000000 19.6 ns/op", + want: &Bench{ + Name: "BenchmarkEncrypt", + N: 100000000, NsOp: 19.6, + Measured: NsOp, + }, + }, + { + line: "BenchmarkEncrypt 100000000 19.6 ns/op 817.77 MB/s", + want: &Bench{ + Name: "BenchmarkEncrypt", + N: 100000000, NsOp: 19.6, MbS: 817.77, + Measured: NsOp | MbS, + }, + }, + { + line: "BenchmarkEncrypt 100000000 19.6 ns/op 817.77", + want: &Bench{ + Name: "BenchmarkEncrypt", + N: 100000000, NsOp: 19.6, + Measured: NsOp, + }, + }, + { + line: "BenchmarkEncrypt 100000000 19.6 ns/op 817.77 MB/s 5 allocs/op", + want: &Bench{ + Name: "BenchmarkEncrypt", + N: 100000000, NsOp: 19.6, MbS: 817.77, AllocsOp: 5, + Measured: NsOp | MbS | AllocsOp, + }, + }, + { + line: "BenchmarkEncrypt 100000000 19.6 ns/op 817.77 MB/s 3 B/op 5 allocs/op", + want: &Bench{ + Name: "BenchmarkEncrypt", + N: 100000000, NsOp: 19.6, MbS: 817.77, BOp: 3, AllocsOp: 5, + Measured: NsOp | MbS | BOp | AllocsOp, + }, + }, + // error handling cases + { + line: "BenchPress 100 19.6 ns/op", // non-benchmark + err: true, + }, + { + line: "BenchmarkEncrypt lots 19.6 ns/op", // non-int iterations + err: true, + }, + { + line: "BenchmarkBridge 100000000 19.6 smoots", // unknown unit + want: &Bench{ + Name: "BenchmarkBridge", + N: 100000000, + }, + }, + { + line: "PASS", + err: true, + }, + } + + for _, tt := range cases { + have, err := ParseLine(tt.line) + if tt.err && err == nil { + t.Errorf("parsing line %q should have failed", tt.line) + continue + } + if !reflect.DeepEqual(have, tt.want) { + t.Errorf("parsed line %q incorrectly, want %v have %v", tt.line, tt.want, have) + } + } +} + +func TestParseBenchSet(t *testing.T) { + // Test two things: + // 1. The noise that can accompany testing.B output gets ignored. + // 2. Benchmarks with the same name have their order preserved. + in := ` + ? crypto [no test files] + PASS + pem_decrypt_test.go:17: test 4. %!s(x509.PEMCipher=5) + ... [output truncated] + + BenchmarkEncrypt 100000000 19.6 ns/op + BenchmarkEncrypt 5000000 517 ns/op + === RUN TestChunk + --- PASS: TestChunk (0.00 seconds) + --- SKIP: TestLinuxSendfile (0.00 seconds) + fs_test.go:716: skipping; linux-only test + BenchmarkReadRequestApachebench 1000000 2960 ns/op 27.70 MB/s 839 B/op 9 allocs/op + BenchmarkClientServerParallel64 50000 59192 ns/op 7028 B/op 60 allocs/op + ok net/http 95.783s + ` + + want := BenchSet{ + "BenchmarkReadRequestApachebench": []*Bench{ + { + Name: "BenchmarkReadRequestApachebench", + N: 1000000, NsOp: 2960, MbS: 27.70, BOp: 839, AllocsOp: 9, + Measured: NsOp | MbS | BOp | AllocsOp, + }, + }, + "BenchmarkClientServerParallel64": []*Bench{ + { + Name: "BenchmarkClientServerParallel64", + N: 50000, NsOp: 59192, BOp: 7028, AllocsOp: 60, + Measured: NsOp | BOp | AllocsOp, + }, + }, + "BenchmarkEncrypt": []*Bench{ + { + Name: "BenchmarkEncrypt", + N: 100000000, NsOp: 19.6, + Measured: NsOp, + }, + { + Name: "BenchmarkEncrypt", + N: 5000000, NsOp: 517, + Measured: NsOp, + }, + }, + } + + have, err := ParseBenchSet(strings.NewReader(in)) + if err != nil { + t.Fatalf("unexpected err during ParseBenchSet: %v", err) + } + if !reflect.DeepEqual(want, have) { + t.Errorf("parsed bench set incorrectly, want %v have %v", want, have) + } +}