зеркало из https://github.com/mozilla/rappor.git
Simplify ./regtest.sh parameters.
This commit is contained in:
Tom Kaitchuck
Родитель
a400c328e3
Коммит
fdaf06faff
12
demo.sh
12
demo.sh
|
|
@ -51,29 +51,23 @@ build() {
|
|||
# Main entry point that is documented in README.md. Uses the Python client.
|
||||
run() {
|
||||
# Run all the test cases that start with "demo".
|
||||
./regtest.sh run-seq '^demo' 1 python
|
||||
./regtest.sh run-seq '^demo' python
|
||||
}
|
||||
|
||||
quick-python() {
|
||||
./regtest.sh run-seq '^demo3' 1 python
|
||||
./regtest.sh run-seq '^demo3' python
|
||||
}
|
||||
|
||||
quick-cpp() {
|
||||
# For now we build it first. Don't want to build it in parallel.
|
||||
./build.sh cpp-client
|
||||
|
||||
./regtest.sh run-seq '^demo3' 1 cpp
|
||||
}
|
||||
|
||||
# NOTE: This behaves worse than 'quick', because of cohort issue.
|
||||
quick-fast-counts() {
|
||||
./regtest.sh run-seq '^demo3' 1 fast_counts
|
||||
./regtest.sh run-seq '^demo3' cpp
|
||||
}
|
||||
|
||||
quick() {
|
||||
quick-python
|
||||
quick-cpp
|
||||
quick-fast-counts
|
||||
}
|
||||
|
||||
# TODO: Port these old bad cases to regtest_spec.py.
|
||||
|
|
|
|||
181
regtest.sh
181
regtest.sh
|
|
@ -8,22 +8,19 @@ echo "
|
|||
At the end, it will print an HTML summary.
|
||||
|
||||
Three main functions are
|
||||
run [[<pattern> [<num> [<fast>]] - run tests matching <pattern> in
|
||||
parallel, each <num> times. The fast
|
||||
mode (T/F) shortcuts generation of
|
||||
reports.
|
||||
run-seq [<pattern> [<num> [<fast>]] - ditto, except that tests are run
|
||||
run [<pattern> [<lang>]] - run tests matching <pattern> in
|
||||
parallel, each <num> times. The language
|
||||
of the client to use.
|
||||
run-seq [<pattern> [<lang>]] - ditto, except that tests are run
|
||||
sequentially
|
||||
run-all [<num>] - run all tests, in parallel, each <num> times
|
||||
run-all - run all tests, in parallel, each <num> times
|
||||
|
||||
Examples:
|
||||
$ ./regtest.sh run-seq unif-small-typical # Sequential run, matches 1 case
|
||||
$ ./regtest.sh run-seq unif-small- 3 F # Sequential, each test is run three
|
||||
times, using slow generation
|
||||
$ ./regtest.sh run-seq unif-small-typical # Run, the unif-small-typical test
|
||||
$ ./regtest.sh run-seq unif-small- # Sequential, the tests containing:
|
||||
# 'unif-small-'
|
||||
$ ./regtest.sh run unif- # Parallel run, matches multiple cases
|
||||
$ ./regtest.sh run unif- 5 # Parallel run, matches multiple cases, each test
|
||||
is run 5 times
|
||||
$ ./regtest.sh run-all # Run all tests once
|
||||
$ ./regtest.sh run-all # Run all tests
|
||||
|
||||
The <pattern> argument is a regex in 'grep -E' format. (Detail: Don't
|
||||
use $ in the pattern, since it matches the whole spec line and not just the
|
||||
|
|
@ -172,92 +169,73 @@ _run-one-instance() {
|
|||
local instance_dir=$case_dir/$test_instance
|
||||
mkdir --verbose -p $instance_dir
|
||||
|
||||
# NOTE: This is a nested case statement on the same variable ($impl). If
|
||||
# it's fast_counts, we just run gen_counts.R. If it's anything else, then we
|
||||
# generate raw reports, run them through the appropriate client, and then sum
|
||||
# the bits.
|
||||
banner "Generating reports (gen_reports.R)"
|
||||
|
||||
# the TRUE_VALUES_PATH environment variable can be used to avoid
|
||||
# generating new values every time. NOTE: You are responsible for making
|
||||
# sure the params match!
|
||||
|
||||
local true_values=${TRUE_VALUES_PATH:-}
|
||||
if test -z "$true_values"; then
|
||||
true_values=$instance_dir/case_true_values.csv
|
||||
tests/gen_true_values.R $distr $num_unique_values $num_clients \
|
||||
$values_per_client $num_cohorts \
|
||||
$true_values
|
||||
else
|
||||
# TEMP hack: Make it visible to plot.
|
||||
# TODO: Fix compare_dist.R
|
||||
ln -s -f --verbose \
|
||||
$PWD/$true_values \
|
||||
$instance_dir/case_true_values.csv
|
||||
fi
|
||||
|
||||
case $impl in
|
||||
fast_counts)
|
||||
local params_file=$case_dir/case_params.csv
|
||||
local true_map_file=$case_dir/case_true_map.csv
|
||||
python)
|
||||
banner "Running RAPPOR Python client"
|
||||
|
||||
banner "Generating counts directly (gen_counts.R)"
|
||||
# Writes encoded "out" file, true histogram, true inputs to
|
||||
# $instance_dir.
|
||||
time tests/rappor_sim.py \
|
||||
--num-bits $num_bits \
|
||||
--num-hashes $num_hashes \
|
||||
--num-cohorts $num_cohorts \
|
||||
-p $p \
|
||||
-q $q \
|
||||
-f $f \
|
||||
< $true_values \
|
||||
> "$instance_dir/case_reports.csv"
|
||||
;;
|
||||
|
||||
cpp)
|
||||
banner "Running RAPPOR C++ client (see rappor_sim.log for errors)"
|
||||
|
||||
time client/cpp/_tmp/rappor_sim \
|
||||
$num_bits \
|
||||
$num_hashes \
|
||||
$num_cohorts \
|
||||
$p \
|
||||
$q \
|
||||
$f \
|
||||
< $true_values \
|
||||
> "$instance_dir/case_reports.csv" \
|
||||
2>"$instance_dir/rappor_sim.log"
|
||||
|
||||
# Writes _counts.csv and _hist.csv
|
||||
tests/gen_counts.R $distr $num_clients $values_per_client $params_file \
|
||||
$true_map_file "$instance_dir/case"
|
||||
;;
|
||||
|
||||
*)
|
||||
banner "Generating reports (gen_reports.R)"
|
||||
|
||||
# the TRUE_VALUES_PATH environment variable can be used to avoid
|
||||
# generating new values every time. NOTE: You are responsible for making
|
||||
# sure the params match!
|
||||
|
||||
local true_values=${TRUE_VALUES_PATH:-}
|
||||
if test -z "$true_values"; then
|
||||
true_values=$instance_dir/case_true_values.csv
|
||||
tests/gen_true_values.R $distr $num_unique_values $num_clients \
|
||||
$values_per_client $num_cohorts \
|
||||
$true_values
|
||||
else
|
||||
# TEMP hack: Make it visible to plot.
|
||||
# TODO: Fix compare_dist.R
|
||||
ln -s -f --verbose \
|
||||
$PWD/$true_values \
|
||||
$instance_dir/case_true_values.csv
|
||||
fi
|
||||
|
||||
case $impl in
|
||||
python)
|
||||
banner "Running RAPPOR Python client"
|
||||
|
||||
# Writes encoded "out" file, true histogram, true inputs to
|
||||
# $instance_dir.
|
||||
time tests/rappor_sim.py \
|
||||
--num-bits $num_bits \
|
||||
--num-hashes $num_hashes \
|
||||
--num-cohorts $num_cohorts \
|
||||
-p $p \
|
||||
-q $q \
|
||||
-f $f \
|
||||
< $true_values \
|
||||
> "$instance_dir/case_reports.csv"
|
||||
;;
|
||||
|
||||
cpp)
|
||||
banner "Running RAPPOR C++ client (see rappor_sim.log for errors)"
|
||||
|
||||
time client/cpp/_tmp/rappor_sim \
|
||||
$num_bits \
|
||||
$num_hashes \
|
||||
$num_cohorts \
|
||||
$p \
|
||||
$q \
|
||||
$f \
|
||||
< $true_values \
|
||||
> "$instance_dir/case_reports.csv" \
|
||||
2>"$instance_dir/rappor_sim.log"
|
||||
|
||||
;;
|
||||
|
||||
*)
|
||||
log "Invalid impl $impl (should be one of fast_counts|python|cpp)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
banner "Summing RAPPOR IRR bits to get 'counts'"
|
||||
|
||||
bin/sum_bits.py \
|
||||
$case_dir/case_params.csv \
|
||||
< $instance_dir/case_reports.csv \
|
||||
> $instance_dir/case_counts.csv
|
||||
log "Invalid impl $impl (should be one of python|cpp)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
banner "Summing RAPPOR IRR bits to get 'counts'"
|
||||
|
||||
bin/sum_bits.py \
|
||||
$case_dir/case_params.csv \
|
||||
< $instance_dir/case_reports.csv \
|
||||
> $instance_dir/case_counts.csv
|
||||
;;
|
||||
|
||||
local out_dir=${instance_dir}_report
|
||||
mkdir --verbose -p $out_dir
|
||||
|
||||
|
|
@ -344,17 +322,16 @@ default-processes() {
|
|||
# Args:
|
||||
# spec_gen: A program to execute to generate the spec.
|
||||
# spec_regex: A pattern selecting the subset of tests to run
|
||||
# instances: A number of times each test case is run
|
||||
# parallel: Whether the tests are run in parallel (T/F). Sequential
|
||||
# runs log to the console; parallel runs log to files.
|
||||
# impl: one of fast_counts, python, cpp
|
||||
# impl: one of python, or cpp
|
||||
|
||||
_run-tests() {
|
||||
local spec_gen=$1
|
||||
local spec_regex="$2" # grep -E format on the spec, can be empty
|
||||
local instances=$3
|
||||
local parallel=$4
|
||||
local impl=$5
|
||||
local parallel=$3
|
||||
local impl=$4
|
||||
local instances=1
|
||||
|
||||
local regtest_dir=$REGTEST_BASE_DIR/$impl
|
||||
rm -r -f --verbose $regtest_dir
|
||||
|
|
@ -403,35 +380,29 @@ readonly REGTEST_SPEC=tests/regtest_spec.py
|
|||
# Run tests sequentially. NOTE: called by demo.sh.
|
||||
run-seq() {
|
||||
local spec_regex=${1:-'^r-'} # grep -E format on the spec
|
||||
local instances=${2:-1}
|
||||
local impl=${3:-fast_counts}
|
||||
local impl=$2
|
||||
|
||||
time _run-tests $REGTEST_SPEC $spec_regex $instances F $impl
|
||||
time _run-tests $REGTEST_SPEC $spec_regex F $impl
|
||||
}
|
||||
|
||||
# Run tests in parallel
|
||||
run() {
|
||||
local spec_regex=${1:-'^r-'} # grep -E format on the spec
|
||||
local instances=${2:-1}
|
||||
local impl=${3:-fast_counts}
|
||||
local impl=$3
|
||||
|
||||
time _run-tests $REGTEST_SPEC $spec_regex $instances T $impl
|
||||
time _run-tests $REGTEST_SPEC $spec_regex T $impl
|
||||
}
|
||||
|
||||
# Run tests in parallel (7+ minutes on 8 cores)
|
||||
run-all() {
|
||||
local instances=${1:-1}
|
||||
|
||||
log "Running all tests. Can take a while."
|
||||
time _run-tests $REGTEST_SPEC '^r-' $instances T fast_counts
|
||||
time _run-tests $REGTEST_SPEC '^r-' T cpp
|
||||
}
|
||||
|
||||
run-user() {
|
||||
local spec_regex=${1:-}
|
||||
local instances=${2:-1}
|
||||
local parallel=T # too much memory
|
||||
time _run-tests tests/user_spec.py "$spec_regex" $instances $parallel \
|
||||
fast_counts
|
||||
time _run-tests tests/user_spec.py "$spec_regex" $parallel cpp
|
||||
}
|
||||
|
||||
# Use stable true values
|
||||
|
|
|
|||
2
test.sh
2
test.sh
|
|
@ -119,8 +119,6 @@ r-unit() {
|
|||
# This one needs to be run from the root dir
|
||||
tests/compare_dist_test.R
|
||||
|
||||
tests/gen_counts_test.R
|
||||
|
||||
tests/gen_true_values_test.R
|
||||
|
||||
analysis/R/decode_test.R
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче