Measure code coverage between tests

Go has code coverage tooling for test mode, which temporarily rewrites
the source code to insert annotations which will activate during the
test run and track progress of executed code. Then, upon process
completion, that information is dumped into a coverage report.

We can't use this approach for hub, at least not without substantial
changes. First of all, hub's test coverage is mostly "from the outside",
utilizing Cucumber to invoke the binary with different arguments and
inspect the outputs and result. There are some tests in go, but they are
minimal compared to the cukes.

Second, hub frequently aborts the process on errors via `os.Exit(1)`,
and those scenarios need to be tested too. However, if the process exits
prematurely, the code coverage report will never be generated.

To work around this, I first used the go tool that annotates the source:

    go tool cover -mode=set -var=LiveCoverage myfile.go

This injects `LiveCoverage.Count[pos] = 1` lines at appropriate places
all over the source code, and generates a mapping of line/column
positions in the original source.

Then I rewrite those lines to become a method invocation:

    coverage.Record(LiveCoverage, pos)

The new `Record` method will immediately append the information to a
code coverage report file as soon as it's invoked. This ensures that
there is coverage information even if the process gets aborted.

This approach works the same for go tests as well as for cukes. They all
append to the same file. Finally, the rest of Go tooling is used to
generate an HTML report of code coverage:

    go tool cover -html=cover.out

coverage/coverage.go

@@ -0,0 +1,53 @@
+package coverage
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"runtime"
+)
+
+var out io.Writer
+var seen map[string]bool
+
+func init() {
+	var err error
+	out, err = os.OpenFile(os.Getenv("HUB_COVERAGE"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		panic(err)
+	}
+	seen = make(map[string]bool)
+}
+
+func Record(data interface{}, i int) {
+	_, filename, _, _ := runtime.Caller(1)
+	if !seen[filename] {
+		seen[filename] = true
+		d := reflect.ValueOf(data)
+		count := reflect.ValueOf(d.FieldByName("Count").Interface())
+		total := count.Len()
+		for j := 0; j < total; j++ {
+			write(data, j, 0, filename)
+		}
+	}
+	write(data, i, 1, filename)
+}
+
+func write(data interface{}, i, count int, filename string) {
+	d := reflect.ValueOf(data)
+	pos := reflect.ValueOf(d.FieldByName("Pos").Interface())
+	numStmt := reflect.ValueOf(d.FieldByName("NumStmt").Interface())
+
+	fmt.Fprintf(
+		out,
+		"%s:%d.%d,%d.%d %d %d\n",
+		filename,
+		pos.Index(3*i).Uint(),
+		pos.Index(3*i+2).Uint()&0xFFFF,
+		pos.Index(3*i+1).Uint(),
+		pos.Index(3*i+2).Uint()>>16&0xFFFF,
+		numStmt.Index(i).Uint(),
+		count,
+	)
+}

script/coverage

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e
+
+source_files() {
+  script/build files | grep -vE '^\./(coverage|fixtures)/'
+}
+
+prepare() {
+  if ! git diff --quiet; then
+    echo "Error: please commit your changes before continuing." >&2
+    exit 1
+  fi
+
+  local n=0
+  for f in $(source_files); do
+    go tool cover -mode=set -var="LiveCoverage$((++n))" "$f" > "$f"~
+    sed -E '
+      /^package /a\
+      import "github.com/github/hub/coverage"
+      s/(LiveCoverage[0-9]+)\.Count\[([0-9]+)\][^;]+/coverage.Record(\1, \2)/
+    ' < "$f"~ > "$f"
+    rm "$f"~
+  done
+
+  rm -rf "$HUB_COVERAGE"
+  mkdir -p "${HUB_COVERAGE%/*}"
+}
+
+generate() {
+  source_files | xargs git checkout --
+
+  echo 'mode: count' > "$HUB_COVERAGE"~
+  sed -E 's!^.+/(github.com/github/hub/)!\1!' "$HUB_COVERAGE" | awk '
+    { a[substr($0, 0, length()-2)] += $(NF) }
+    END { for (k in a) print k, a[k] }
+  ' >> "$HUB_COVERAGE"~
+
+  go tool cover -func="$HUB_COVERAGE"~ > "${HUB_COVERAGE%.out}.func"
+  if [ -z "$CI" ]; then
+    go tool cover -html="$HUB_COVERAGE"~ -o "${HUB_COVERAGE%.out}.html"
+  fi
+
+  awk '/^total:/ { print $(NF) }' "${HUB_COVERAGE%.out}.func"
+}
+
+case "${1?}" in
+  prepare | generate )
+    "$1"
+    ;;
+  * )
+    exit 1
+    ;;
+esac

script/test

@@ -1,30 +1,55 @@
 #!/usr/bin/env bash
-# Usage: script/test
+# Usage: script/test [--coverage [<MIN>]]
 #
 # Run Go and Cucumber test suites for hub.
 
 set -e
 
-case "$1" in
-"" )
-  ;;
--h | --help )
-  sed -ne '/^#/!q;s/.\{1,2\}//;1d;p' < "$0"
-  exit
-  ;;
-* )
-  "$0" --help >&2
-  exit 1
-esac
+while [ $# -gt 0 ]; do
+  case "$1" in
+  --coverage )
+    export HUB_COVERAGE="$PWD/tmp/cover.out"
+    if [ "$2" -gt 0 ] 2>/dev/null; then
+      min_coverage="$2"
+      shift 2
+    else
+      min_coverage=1
+      shift 1
+    fi
+    ;;
+  -h | --help )
+    sed -ne '/^#/!q;s/.\{1,2\}//;1d;p' < "$0"
+    exit
+    ;;
+  * )
+    "$0" --help >&2
+    exit 1
+  esac
+done
 
 STATUS=0
 
 trap "exit 1" INT
 
+[ -z "$HUB_COVERAGE" ] || script/coverage prepare
 script/build
 script/build test || STATUS="$?"
 script/ruby-test || STATUS="$?"
 
+if [ -n "$HUB_COVERAGE" ]; then
+  total_coverage="$(script/coverage generate)"
+  echo "Code coverage: $total_coverage"
+  if [ "${total_coverage%.*}" -lt "$min_coverage" ]; then
+    echo "Error: coverage dropped below the minimum treshold of ${min_coverage}%!"
+    if [ -n "$CI" ]; then
+      html_result="${HUB_COVERAGE%.out}.html"
+      html_result="${html_result#$PWD/}"
+      printf 'Please run `script/test --coverage` locally and open `%s` to analyze the results.\n' "$html_result"
+    fi
+    STATUS=1
+  fi
+fi
+
 if [ -n "$CI" ]; then
   make fmt >/dev/null
   if ! git diff -U1 --exit-code; then