diff --git a/t/perf/bisect_regression b/t/perf/bisect_regression new file mode 100755 index 0000000000..a94d9955d0 --- /dev/null +++ b/t/perf/bisect_regression @@ -0,0 +1,73 @@ +#!/bin/sh + +# Read a line coming from `./aggregate.perl --sort-by regression ...` +# and automatically bisect to find the commit responsible for the +# performance regression. +# +# Lines from `./aggregate.perl --sort-by regression ...` look like: +# +# +100.0% p7821-grep-engines-fixed.1 0.04(0.10+0.03) 0.08(0.11+0.08) v2.14.3 v2.15.1 +# +33.3% p7820-grep-engines.1 0.03(0.08+0.02) 0.04(0.08+0.02) v2.14.3 v2.15.1 +# + +die () { + echo >&2 "error: $*" + exit 1 +} + +while [ $# -gt 0 ]; do + arg="$1" + case "$arg" in + --help) + echo "usage: $0 [--config file] [--subsection subsection]" + exit 0 + ;; + --config) + shift + GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1") + export GIT_PERF_CONFIG_FILE + shift ;; + --subsection) + shift + GIT_PERF_SUBSECTION="$1" + export GIT_PERF_SUBSECTION + shift ;; + --*) + die "unrecognised option: '$arg'" ;; + *) + die "unknown argument '$arg'" + ;; + esac +done + +read -r regression subtest oldtime newtime oldrev newrev + +test_script=$(echo "$subtest" | sed -e 's/\(.*\)\.[0-9]*$/\1.sh/') +test_number=$(echo "$subtest" | sed -e 's/.*\.\([0-9]*\)$/\1/') + +# oldtime and newtime are decimal number, not integers + +oldtime=$(echo "$oldtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/') +newtime=$(echo "$newtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/') + +test $(echo "$newtime" "$oldtime" | awk '{ print ($1 > $2) }') = 1 || + die "New time '$newtime' shoud be greater than old time '$oldtime'" + +tmpdir=$(mktemp -d -t bisect_regression_XXXXXX) || die "Failed to create temp directory" +echo "$oldtime" >"$tmpdir/oldtime" || die "Failed to write to '$tmpdir/oldtime'" +echo "$newtime" >"$tmpdir/newtime" || die "Failed to write to '$tmpdir/newtime'" + +# Bisecting must be performed from the top level directory (even with --no-checkout) +( + toplevel_dir=$(git rev-parse --show-toplevel) || die "Failed to find top level directory" + cd "$toplevel_dir" || die "Failed to cd into top level directory '$toplevel_dir'" + + git bisect start --no-checkout "$newrev" "$oldrev" || die "Failed to start bisecting" + + git bisect run t/perf/bisect_run_script "$test_script" "$test_number" "$tmpdir" + res="$?" + + git bisect reset + + exit "$res" +) diff --git a/t/perf/bisect_run_script b/t/perf/bisect_run_script new file mode 100755 index 0000000000..038255df4b --- /dev/null +++ b/t/perf/bisect_run_script @@ -0,0 +1,47 @@ +#!/bin/sh + +script="$1" +test_number="$2" +info_dir="$3" + +# This aborts the bisection immediately +die () { + echo >&2 "error: $*" + exit 255 +} + +bisect_head=$(git rev-parse --verify BISECT_HEAD) || die "Failed to find BISECT_HEAD ref" + +script_number=$(echo "$script" | sed -e "s/^p\([0-9]*\).*\$/\1/") || die "Failed to get script number for '$script'" + +oldtime=$(cat "$info_dir/oldtime") || die "Failed to access '$info_dir/oldtime'" +newtime=$(cat "$info_dir/newtime") || die "Failed to access '$info_dir/newtime'" + +cd t/perf || die "Failed to cd into 't/perf'" + +result_file="$info_dir/perf_${script_number}_${bisect_head}_results.txt" + +GIT_PERF_DIRS_OR_REVS="$bisect_head" +export GIT_PERF_DIRS_OR_REVS + +./run "$script" >"$result_file" 2>&1 || die "Failed to run perf test '$script'" + +rtime=$(sed -n "s/^$script_number\.$test_number:.*\([0-9]\+\.[0-9]\+\)(.*).*\$/\1/p" "$result_file") + +echo "newtime: $newtime" +echo "rtime: $rtime" +echo "oldtime: $oldtime" + +# Compare ($newtime - $rtime) with ($rtime - $oldtime) +# Times are decimal number, not integers + +if test $(echo "$newtime" "$rtime" "$oldtime" | awk '{ print ($1 - $2 > $2 - $3) }') = 1 +then + # Current commit is considered "good/old" + echo "$rtime" >"$info_dir/oldtime" + exit 0 +else + # Current commit is considered "bad/new" + echo "$rtime" >"$info_dir/newtime" + exit 1 +fi