зеркало из https://github.com/mozilla/rappor.git
465 строки
13 KiB
Bash
Executable File
465 строки
13 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Run end-to-end tests in parallel.
|
|
#
|
|
# Usage:
|
|
# ./regtest.sh <function name>
|
|
|
|
# At the end, it will print an HTML summary.
|
|
#
|
|
# Three main functions are
|
|
# run [[<pattern> [<num> [<fast>]] - run tests matching <pattern> in
|
|
# parallel, each <num> times. The fast
|
|
# mode (T/F) shortcuts generation of
|
|
# reports.
|
|
# run-seq [<pattern> [<num> [<fast>]] - ditto, except that tests are run
|
|
# sequentially
|
|
# run-all [<num>] - run all tests, in parallel, each <num> times
|
|
#
|
|
# Examples:
|
|
# $ ./regtest.sh run-seq unif-small-typical # Sequential run, matches 1 case
|
|
# $ ./regtest.sh run-seq unif-small- 3 F # Sequential, each test is run three
|
|
# times, using slow generation
|
|
# $ ./regtest.sh run unif- # Parallel run, matches multiple cases
|
|
# $ ./regtest.sh run unif- 5 # Parallel run, matches multiple cases, each test
|
|
# is run 5 times
|
|
# $ ./regtest.sh run-all # Run all tests once
|
|
#
|
|
# The <pattern> argument is a regex in 'grep -E' format. (Detail: Don't
|
|
# use $ in the pattern, since it matches the whole spec line and not just the
|
|
# test case name.) The number of processors used in a parallel run is one less
|
|
# than the number of CPUs on the machine.
|
|
|
|
|
|
# Future speedups:
|
|
# - Reuse the same input -- come up with naming scheme based on params
|
|
# - Reuse the same maps -- ditto, rappor library can cache it
|
|
|
|
set -o nounset
|
|
set -o pipefail
|
|
set -o errexit
|
|
|
|
. util.sh
|
|
|
|
readonly THIS_DIR=$(dirname $0)
|
|
readonly REPO_ROOT=$THIS_DIR
|
|
readonly CLIENT_DIR=$REPO_ROOT/client/python
|
|
# subdirs are in _tmp/$impl, which shouldn't overlap with anything else in _tmp
|
|
readonly REGTEST_BASE_DIR=_tmp
|
|
|
|
# All the Python tools need this
|
|
export PYTHONPATH=$CLIENT_DIR
|
|
|
|
print-unique-values() {
|
|
local num_unique_values=$1
|
|
seq 1 $num_unique_values | awk '{print "v" $1}'
|
|
}
|
|
|
|
# Add some more candidates here. We hope these are estimated at 0.
|
|
# e.g. if add_start=51, and num_additional is 20, show v51-v70
|
|
more-candidates() {
|
|
local last_true=$1
|
|
local num_additional=$2
|
|
|
|
local begin
|
|
local end
|
|
begin=$(expr $last_true + 1)
|
|
end=$(expr $last_true + $num_additional)
|
|
|
|
seq $begin $end | awk '{print "v" $1}'
|
|
}
|
|
|
|
# Args:
|
|
# unique_values: File of unique true values
|
|
# last_true: last true input, e.g. 50 if we generated "v1" .. "v50".
|
|
# num_additional: additional candidates to generate (starting at 'last_true')
|
|
# to_remove: Regex of true values to omit from the candidates list, or the
|
|
# string 'NONE' if none should be. (Our values look like 'v1', 'v2', etc. so
|
|
# there isn't any ambiguity.)
|
|
print-candidates() {
|
|
local unique_values=$1
|
|
local last_true=$2
|
|
local num_additional=$3
|
|
local to_remove=$4
|
|
|
|
if test $to_remove = NONE; then
|
|
cat $unique_values # include all true inputs
|
|
else
|
|
egrep -v $to_remove $unique_values # remove some true inputs
|
|
fi
|
|
more-candidates $last_true $num_additional
|
|
}
|
|
|
|
# Generate a single test case, specified by a line of the test spec.
|
|
# This is a helper function for _run_tests().
|
|
_setup-one-case() {
|
|
local impl=$1
|
|
shift # impl is not part of the spec; the next 13 params are
|
|
|
|
local test_case=$1
|
|
|
|
# input params
|
|
local dist=$2
|
|
local num_unique_values=$3
|
|
local num_clients=$4
|
|
local values_per_client=$5
|
|
|
|
# RAPPOR params
|
|
local num_bits=$6
|
|
local num_hashes=$7
|
|
local num_cohorts=$8
|
|
local p=$9
|
|
local q=${10} # need curly braces to get the 10th arg
|
|
local f=${11}
|
|
|
|
# map params
|
|
local num_additional=${12}
|
|
local to_remove=${13}
|
|
|
|
banner 'Setting up parameters and candidate files for '$test_case
|
|
|
|
local case_dir=$REGTEST_BASE_DIR/$impl/$test_case
|
|
mkdir --verbose -p $case_dir
|
|
|
|
# Save the "spec"
|
|
echo "$@" > $case_dir/spec.txt
|
|
|
|
local params_path=$case_dir/case_params.csv
|
|
|
|
echo 'k,h,m,p,q,f' > $params_path
|
|
echo "$num_bits,$num_hashes,$num_cohorts,$p,$q,$f" >> $params_path
|
|
|
|
print-unique-values $num_unique_values > $case_dir/case_unique_values.txt
|
|
|
|
local true_map_path=$case_dir/case_true_map.csv
|
|
|
|
analysis/tools/hash_candidates.py \
|
|
$params_path \
|
|
< $case_dir/case_unique_values.txt \
|
|
> $true_map_path
|
|
|
|
# banner "Constructing candidates"
|
|
|
|
print-candidates \
|
|
$case_dir/case_unique_values.txt $num_unique_values \
|
|
$num_additional "$to_remove" \
|
|
> $case_dir/case_candidates.txt
|
|
|
|
# banner "Hashing candidates to get 'map'"
|
|
|
|
analysis/tools/hash_candidates.py \
|
|
$params_path \
|
|
< $case_dir/case_candidates.txt \
|
|
> $case_dir/case_map.csv
|
|
}
|
|
|
|
# Run a single test instance, specified by <test_name, instance_num>.
|
|
# This is a helper function for _run_tests().
|
|
_run-one-instance() {
|
|
local test_case=$1
|
|
local test_instance=$2
|
|
local impl=$3
|
|
|
|
local case_dir=$REGTEST_BASE_DIR/$impl/$test_case
|
|
|
|
read -r \
|
|
case_name distr num_unique_values num_clients values_per_client \
|
|
num_bits num_hashes num_cohorts p q f \
|
|
num_additional to_remove \
|
|
< $case_dir/spec.txt
|
|
|
|
local instance_dir=$case_dir/$test_instance
|
|
mkdir --verbose -p $instance_dir
|
|
|
|
# NOTE: This is a nested case statement on the same variable ($impl). If
|
|
# it's fast_counts, we just run gen_counts.R. If it's anything else, then we
|
|
# generate raw reports, run them through the appropriate client, and then sum
|
|
# the bits.
|
|
|
|
case $impl in
|
|
fast_counts)
|
|
local params_file=$case_dir/case_params.csv
|
|
local true_map_file=$case_dir/case_true_map.csv
|
|
|
|
banner "Generating counts directly (gen_counts.R)"
|
|
|
|
# Writes _counts.csv and _hist.csv
|
|
tests/gen_counts.R $distr $num_clients $values_per_client $params_file \
|
|
$true_map_file "$instance_dir/case"
|
|
;;
|
|
|
|
*)
|
|
banner "Generating reports (gen_reports.R)"
|
|
|
|
# the TRUE_VALUES_PATH environment variable can be used to avoid
|
|
# generating new values every time. NOTE: You are responsible for making
|
|
# sure the params match!
|
|
|
|
local true_values=${TRUE_VALUES_PATH:-}
|
|
if test -z "$true_values"; then
|
|
true_values=$instance_dir/case_true_values.csv
|
|
tests/gen_true_values.R $distr $num_unique_values $num_clients \
|
|
$values_per_client $num_cohorts \
|
|
$true_values
|
|
else
|
|
# TEMP hack: Make it visible to plot.
|
|
# TODO: Fix compare_dist.R
|
|
ln -s -f --verbose \
|
|
$PWD/$true_values \
|
|
$instance_dir/case_true_values.csv
|
|
fi
|
|
|
|
case $impl in
|
|
python)
|
|
banner "Running RAPPOR Python client"
|
|
|
|
# Writes encoded "out" file, true histogram, true inputs to
|
|
# $instance_dir.
|
|
time tests/rappor_sim.py \
|
|
--num-bits $num_bits \
|
|
--num-hashes $num_hashes \
|
|
--num-cohorts $num_cohorts \
|
|
-p $p \
|
|
-q $q \
|
|
-f $f \
|
|
< $true_values \
|
|
> "$instance_dir/case_reports.csv"
|
|
;;
|
|
|
|
cpp)
|
|
banner "Running RAPPOR C++ client (see rappor_sim.log for errors)"
|
|
|
|
time client/cpp/_tmp/rappor_sim \
|
|
$num_bits \
|
|
$num_hashes \
|
|
$num_cohorts \
|
|
$p \
|
|
$q \
|
|
$f \
|
|
< $true_values \
|
|
> "$instance_dir/case_reports.csv" \
|
|
2>"$instance_dir/rappor_sim.log"
|
|
|
|
;;
|
|
|
|
*)
|
|
log "Invalid impl $impl (should be one of fast_counts|python|cpp)"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
banner "Summing RAPPOR IRR bits to get 'counts'"
|
|
|
|
analysis/tools/sum_bits.py \
|
|
$case_dir/case_params.csv \
|
|
< $instance_dir/case_reports.csv \
|
|
> $instance_dir/case_counts.csv
|
|
;;
|
|
esac
|
|
|
|
local out_dir=${instance_dir}_report
|
|
mkdir --verbose -p $out_dir
|
|
|
|
# Currently, the summary file shows and aggregates timing of the inference
|
|
# engine, which excludes R's loading time and reading of the (possibly
|
|
# substantial) map file. Timing below is more inclusive.
|
|
TIMEFORMAT='Running compare_dist.R took %R seconds'
|
|
time {
|
|
# Input prefix, output dir
|
|
tests/compare_dist.R -t "Test case: $test_case (instance $test_instance)" \
|
|
"$case_dir/case" "$instance_dir/case" $out_dir
|
|
}
|
|
}
|
|
|
|
# Like _run-once-case, but log to a file.
|
|
_run-one-instance-logged() {
|
|
local test_case=$1
|
|
local test_instance=$2
|
|
local impl=$3
|
|
|
|
local log_dir=$REGTEST_BASE_DIR/$impl/$test_case/${test_instance}_report
|
|
mkdir --verbose -p $log_dir
|
|
|
|
log "Started '$test_case' (instance $test_instance) -- logging to $log_dir/log.txt"
|
|
_run-one-instance "$@" >$log_dir/log.txt 2>&1 \
|
|
&& log "Test case $test_case (instance $test_instance) done" \
|
|
|| log "Test case $test_case (instance $test_instance) failed"
|
|
}
|
|
|
|
make-summary() {
|
|
local dir=$1
|
|
local impl=$2
|
|
|
|
local filename=results.html
|
|
|
|
tests/make_summary.py $dir $dir/rows.html
|
|
|
|
pushd $dir >/dev/null
|
|
|
|
cat ../../tests/regtest.html \
|
|
| sed -e '/__TABLE_ROWS__/ r rows.html' -e "s/_IMPL_/$impl/g" \
|
|
> $filename
|
|
|
|
popd >/dev/null
|
|
|
|
log "Wrote $dir/$filename"
|
|
log "URL: file://$PWD/$dir/$filename"
|
|
}
|
|
|
|
test-error() {
|
|
local spec_regex=${1:-}
|
|
log "Some test cases failed"
|
|
if test -n "$spec_regex"; then
|
|
log "(Perhaps none matched pattern '$spec_regex')"
|
|
fi
|
|
# don't quit just yet
|
|
# exit 1
|
|
}
|
|
|
|
# Assuming the spec file, write a list of test case names (first column) with
|
|
# the instance ids (second column), where instance ids run from 1 to $1.
|
|
# Third column is impl.
|
|
_setup-test-instances() {
|
|
local instances=$1
|
|
local impl=$2
|
|
|
|
while read line; do
|
|
for i in $(seq 1 $instances); do
|
|
read case_name _ <<< $line # extract the first token
|
|
echo $case_name $i $impl
|
|
done
|
|
done
|
|
}
|
|
|
|
# Print the default number of parallel processes, which is max(#CPUs - 1, 1)
|
|
default-processes() {
|
|
processors=$(grep -c ^processor /proc/cpuinfo || echo 4) # Linux-specific
|
|
if test $processors -gt 1; then # leave one CPU for the OS
|
|
processors=$(expr $processors - 1)
|
|
fi
|
|
echo $processors
|
|
}
|
|
|
|
# Args:
|
|
# spec_gen: A program to execute to generate the spec.
|
|
# spec_regex: A pattern selecting the subset of tests to run
|
|
# instances: A number of times each test case is run
|
|
# parallel: Whether the tests are run in parallel (T/F). Sequential
|
|
# runs log to the console; parallel runs log to files.
|
|
# impl: one of fast_counts, python, cpp
|
|
|
|
_run-tests() {
|
|
local spec_gen=$1
|
|
local spec_regex="$2" # grep -E format on the spec, can be empty
|
|
local instances=$3
|
|
local parallel=$4
|
|
local impl=$5
|
|
|
|
local regtest_dir=$REGTEST_BASE_DIR/$impl
|
|
rm -r -f --verbose $regtest_dir
|
|
|
|
mkdir --verbose -p $regtest_dir
|
|
|
|
local func
|
|
local processors
|
|
|
|
if test $parallel = F; then
|
|
func=_run-one-instance # output to the console
|
|
processors=1
|
|
else
|
|
func=_run-one-instance-logged
|
|
# Let the user override with MAX_PROC, in case they don't have enough
|
|
# memory.
|
|
processors=${MAX_PROC:-$(default-processes)}
|
|
log "Running $processors parallel processes"
|
|
fi
|
|
|
|
local cases_list=$regtest_dir/test-cases.txt
|
|
# Need -- for regexes that start with -
|
|
$spec_gen | grep -E -- "$spec_regex" > $cases_list
|
|
|
|
# Generate parameters for all test cases.
|
|
cat $cases_list \
|
|
| xargs -l -P $processors -- $0 _setup-one-case $impl \
|
|
|| test-error
|
|
|
|
log "Done generating parameters for all test cases"
|
|
|
|
local instances_list=$regtest_dir/test-instances.txt
|
|
_setup-test-instances $instances $impl < $cases_list > $instances_list
|
|
|
|
cat $instances_list \
|
|
| xargs -l -P $processors -- $0 $func || test-error
|
|
|
|
log "Done running all test instances"
|
|
|
|
make-summary $regtest_dir $impl
|
|
}
|
|
|
|
# used for most tests
|
|
readonly REGTEST_SPEC=tests/regtest_spec.py
|
|
|
|
# Run tests sequentially. NOTE: called by demo.sh.
|
|
run-seq() {
|
|
local spec_regex=${1:-'^r-'} # grep -E format on the spec
|
|
local instances=${2:-1}
|
|
local impl=${3:-fast_counts}
|
|
|
|
time _run-tests $REGTEST_SPEC $spec_regex $instances F $impl
|
|
}
|
|
|
|
# Run tests in parallel
|
|
run() {
|
|
local spec_regex=${1:-'^r-'} # grep -E format on the spec
|
|
local instances=${2:-1}
|
|
local impl=${3:-fast_counts}
|
|
|
|
time _run-tests $REGTEST_SPEC $spec_regex $instances T $impl
|
|
}
|
|
|
|
# Run tests in parallel (7+ minutes on 8 cores)
|
|
run-all() {
|
|
local instances=${1:-1}
|
|
|
|
log "Running all tests. Can take a while."
|
|
time _run-tests $REGTEST_SPEC '^r-' $instances T fast_counts
|
|
}
|
|
|
|
run-user() {
|
|
local spec_regex=${1:-}
|
|
local instances=${2:-1}
|
|
local parallel=T # too much memory
|
|
time _run-tests tests/user_spec.py "$spec_regex" $instances $parallel \
|
|
fast_counts
|
|
}
|
|
|
|
# Use stable true values
|
|
compare-python-cpp() {
|
|
local num_unique_values=100
|
|
local num_clients=10000
|
|
local values_per_client=10
|
|
local num_cohorts=64
|
|
|
|
local true_values=$REGTEST_BASE_DIR/stable_true_values.csv
|
|
|
|
tests/gen_true_values.R \
|
|
exp $num_unique_values $num_clients $values_per_client $num_cohorts \
|
|
$true_values
|
|
|
|
wc -l $true_values
|
|
|
|
# Run Python and C++ simulation on the same input
|
|
|
|
./build.sh cpp-client
|
|
|
|
TRUE_VALUES_PATH=$true_values \
|
|
./regtest.sh run-seq '^demo3' 1 python
|
|
|
|
TRUE_VALUES_PATH=$true_values \
|
|
./regtest.sh run-seq '^demo3' 1 cpp
|
|
|
|
head _tmp/{python,cpp}/demo3/1/case_reports.csv
|
|
}
|
|
|
|
"$@"
|