Added command line spew in the --dry-run option for TestDriver and also refactored common code across run-test scripts for different tests into a run-test-common script
This commit is contained in:
Amit Agarwal
Родитель
547015acf3
Коммит
08119b838e
|
|
@ -1,41 +1,18 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
ConfigDir=$TEST_DIR/../..
|
||||
LogFileName=stderr
|
||||
|
||||
# cntkmpirun <MPI args> <CNTK config file name> <additional CNTK args>
|
||||
if cntkmpirun "-n 4" SimpleMultiGPU.config "precision=double SimpleMultiGPU=[SGD=[ParallelTrain=[DataParallelSGD=[gradientBits=64]]]]" ; then
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank2
|
||||
sed 's/^/MPI Rank 3: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank3
|
||||
exit $ExitCode
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
exit $?
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/../../SimpleMultiGPU.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
fi
|
||||
|
||||
# Since we use the MS MPI program on Windows, the CNTK binary path argument
|
||||
# passed to mpiexec must be in the windows format
|
||||
CNTKBinaryPath=$TEST_CNTK_BINARY
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
CNTKBinaryPath=$(cygpath -aw $CNTKBinaryPath)
|
||||
fi
|
||||
|
||||
MPI_ARGS="-n 4"
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID stderr=$RunDir/stderr precision=double SimpleMultiGPU=[SGD=[ParallelTrain=[DataParallelSGD=[gradientBits=64]]]]"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running "$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
"$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank2
|
||||
sed 's/^/MPI Rank 3: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank3
|
||||
exit $ExitCode
|
||||
|
|
|
|||
|
|
@ -1,41 +1,18 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
ConfigDir=$TEST_DIR/../..
|
||||
LogFileName=stderr
|
||||
|
||||
# cntkmpirun <MPI args> <CNTK config file name> <additional CNTK args>
|
||||
if cntkmpirun "-n 4" SimpleMultiGPU.config "precision=float SimpleMultiGPU=[SGD=[ParallelTrain=[DataParallelSGD=[gradientBits=32]]]]" ; then
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank2
|
||||
sed 's/^/MPI Rank 3: /' $TEST_RUN_DIR/"$LogFileName"_SimpleMultiGPU.logrank3
|
||||
exit $ExitCode
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
exit $?
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/../../SimpleMultiGPU.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
fi
|
||||
|
||||
# Since we use the MS MPI program on Windows, the CNTK binary path argument
|
||||
# passed to mpiexec must be in the windows format
|
||||
CNTKBinaryPath=$TEST_CNTK_BINARY
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
CNTKBinaryPath=$(cygpath -aw $CNTKBinaryPath)
|
||||
fi
|
||||
|
||||
MPI_ARGS="-n 4"
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID stderr=$RunDir/stderr precision=float SimpleMultiGPU=[SGD=[ParallelTrain=[DataParallelSGD=[gradientBits=32]]]]"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running "$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
"$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank2
|
||||
sed 's/^/MPI Rank 3: /' $TEST_RUN_DIR/stderr_SimpleMultiGPU.logrank3
|
||||
exit $ExitCode
|
||||
|
|
|
|||
|
|
@ -1,27 +1,8 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/cntk_dpt.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
#!/bin/bash
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
fi
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running $TEST_CNTK_BINARY $CNTK_ARGS
|
||||
$TEST_CNTK_BINARY $CNTK_ARGS || exit $?
|
||||
# cntkrun <CNTK config file name> <additional CNTK args>
|
||||
cntkrun cntk_dpt.config || exit $?
|
||||
|
|
|
|||
|
|
@ -1,40 +1,17 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
ConfigDir=$TEST_DIR/..
|
||||
LogFileName=stderr
|
||||
|
||||
# cntkmpirun <MPI args> <CNTK config file name> <additional CNTK args>
|
||||
if cntkmpirun "-n 3" cntk.config "precision=double speechTrain=[SGD=[ParallelTrain=[DataParallelSGD=[gradientBits=1]]]] speechTrain=[SGD=[ParallelTrain=[parallelizationStartEpoch=2]]]" ; then
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank2
|
||||
exit $ExitCode
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
exit $?
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/../cntk.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
fi
|
||||
|
||||
# Since we use the MS MPI program on Windows, the CNTK binary path argument
|
||||
# passed to mpiexec must be in the windows format
|
||||
CNTKBinaryPath=$TEST_CNTK_BINARY
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
CNTKBinaryPath=$(cygpath -aw $CNTKBinaryPath)
|
||||
fi
|
||||
|
||||
MPI_ARGS="-n 3"
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID stderr=$RunDir/stderr precision=double speechTrain=[SGD=[ParallelTrain=[DataParallelSGD=[gradientBits=1]]]] speechTrain=[SGD=[ParallelTrain=[parallelizationStartEpoch=2]]]"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running "$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
"$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/stderr_speechTrain.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/stderr_speechTrain.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/stderr_speechTrain.logrank2
|
||||
exit $ExitCode
|
||||
|
|
|
|||
|
|
@ -1,40 +1,17 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
ConfigDir=$TEST_DIR/..
|
||||
LogFileName=stderr
|
||||
|
||||
# cntkmpirun <MPI args> <CNTK config file name> <additional CNTK args>
|
||||
if cntkmpirun "-n 3" cntk.config ; then
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank2
|
||||
exit $ExitCode
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
exit $?
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/../cntk.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
fi
|
||||
|
||||
# Since we use the MS MPI program on Windows, the CNTK binary path argument
|
||||
# passed to mpiexec must be in the windows format
|
||||
CNTKBinaryPath=$TEST_CNTK_BINARY
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
CNTKBinaryPath=$(cygpath -aw $CNTKBinaryPath)
|
||||
fi
|
||||
|
||||
MPI_ARGS="-n 3"
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID stderr=$RunDir/stderr"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running "$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
"$MPI_BINARY" $MPI_ARGS $CNTKBinaryPath $CNTK_ARGS
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/stderr_speechTrain.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/stderr_speechTrain.logrank1
|
||||
sed 's/^/MPI Rank 2: /' $TEST_RUN_DIR/stderr_speechTrain.logrank2
|
||||
exit $ExitCode
|
||||
|
|
|
|||
|
|
@ -1,29 +1,8 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/../cntk.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
NDLDir=$TEST_DIR/..
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
NDLDir=$(cygpath -aw $NDLDir)
|
||||
fi
|
||||
ConfigDir=$TEST_DIR/..
|
||||
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID NDLDir=$NDLDir Truncated=false speechTrain=[reader=[nbruttsineachrecurrentiter=1]] speechTrain=[SGD=[epochSize=2560]] speechTrain=[SGD=[maxEpochs=2]] speechTrain=[SGD=[numMBsToShowResult=1]]"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running $TEST_CNTK_BINARY $CNTK_ARGS
|
||||
$TEST_CNTK_BINARY $CNTK_ARGS || exit $?
|
||||
# cntkrun <CNTK config file name> <additional CNTK args>
|
||||
cntkrun cntk.config 'Truncated=false speechTrain=[reader=[nbruttsineachrecurrentiter=1]] speechTrain=[SGD=[epochSize=2560]] speechTrain=[SGD=[maxEpochs=2]] speechTrain=[SGD=[numMBsToShowResult=1]]' || exit $?
|
||||
|
|
|
|||
|
|
@ -1,29 +1,8 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/../cntk.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
NDLDir=$TEST_DIR/..
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
NDLDir=$(cygpath -aw $NDLDir)
|
||||
fi
|
||||
ConfigDir=$TEST_DIR/..
|
||||
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID NDLDir=$NDLDir"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running $TEST_CNTK_BINARY $CNTK_ARGS
|
||||
$TEST_CNTK_BINARY $CNTK_ARGS || exit $?
|
||||
# cntkrun <CNTK config file name> <additional CNTK args>
|
||||
cntkrun cntk.config || exit $?
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ speechTrain=[
|
|||
traceLevel=1
|
||||
|
||||
#NDLNetworkBuilder=[
|
||||
# networkDescription=$NDLDir$/lstmp-3layer_WithSelfStab.ndl
|
||||
# networkDescription=$ConfigDir$/lstmp-3layer_WithSelfStab.ndl
|
||||
#]
|
||||
|
||||
SGD=[
|
||||
|
|
|
|||
|
|
@ -1,31 +1,12 @@
|
|||
#!/bin/bash
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTK_DEVICE_ID=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTK_DEVICE_ID=0
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
configFile=$TEST_DIR/cntk.config
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
configFile=$(cygpath -aw $configFile)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
fi
|
||||
|
||||
CNTK_ARGS="configFile=$configFile RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTK_DEVICE_ID"
|
||||
MODELS_DIR=$TEST_RUN_DIR/models
|
||||
[ -d $MODELS_DIR ] && rm -rf $MODELS_DIR
|
||||
mkdir -p $MODELS_DIR || exit $?
|
||||
echo === Running $TEST_CNTK_BINARY $CNTK_ARGS
|
||||
$TEST_CNTK_BINARY $CNTK_ARGS || exit $?
|
||||
# cntkrun <CNTK config file name> <additional CNTK args>
|
||||
cntkrun cntk.config || exit $?
|
||||
echo === Deleting last epoch data
|
||||
rm $TEST_RUN_DIR/models/*.dnn
|
||||
echo ==== Re-running from checkpoint
|
||||
$TEST_CNTK_BINARY $CNTK_ARGS || exit $?
|
||||
DeleteExistingModels=0
|
||||
# cntkrun <CNTK config file name> <additional CNTK args>
|
||||
cntkrun cntk.config || exit $?
|
||||
|
|
|
|||
|
|
@ -262,6 +262,10 @@ class Test:
|
|||
if args.verbose:
|
||||
print self.fullName + ": " + line
|
||||
|
||||
if args.dry_run:
|
||||
print line
|
||||
continue
|
||||
|
||||
print >>output, line
|
||||
allLines.append(line)
|
||||
output.flush()
|
||||
|
|
@ -271,13 +275,17 @@ class Test:
|
|||
exitCode = process.wait()
|
||||
success = True
|
||||
|
||||
# checking exit code
|
||||
if exitCode != 0:
|
||||
return TestRunResult.fatalError("Exit code must be 0", "==> got exit code {0} when running: {1}".format(exitCode, " ".join(cmdLine)), logFile = logFile)
|
||||
|
||||
# saving log file path, so it can be reported later
|
||||
result.logFile = logFile
|
||||
|
||||
# checking exit code
|
||||
if exitCode != 0:
|
||||
if args.dry_run:
|
||||
print "[SKIPPED]"
|
||||
return result
|
||||
else:
|
||||
return TestRunResult.fatalError("Exit code must be 0", "==> got exit code {0} when running: {1}".format(exitCode, " ".join(cmdLine)), logFile = logFile)
|
||||
|
||||
# finalizing verification - need to check whether we have any unmatched lines
|
||||
for testCaseRunResult in result.testCaseRunResults:
|
||||
testCaseRunResult.testCase.finalize(testCaseRunResult)
|
||||
|
|
@ -546,6 +554,8 @@ def runCommand(args):
|
|||
devices = args.devices
|
||||
flavors = args.flavors
|
||||
|
||||
os.environ["TEST_ROOT_DIR"] = os.path.dirname(os.path.realpath(sys.argv[0]))
|
||||
|
||||
print "CNTK Test Driver is started"
|
||||
print "Running tests: ", " ".join([y.fullName for y in testsToRun])
|
||||
print "Build location: ", args.build_location
|
||||
|
|
@ -555,6 +565,8 @@ def runCommand(args):
|
|||
if (args.update_baseline):
|
||||
print "*** Running in automatic baseline update mode ***"
|
||||
print ""
|
||||
if args.dry_run:
|
||||
os.environ["DRY_RUN"] = "1"
|
||||
succeededCount, totalCount = 0, 0
|
||||
for test in testsToRun:
|
||||
for flavor in flavors:
|
||||
|
|
@ -569,7 +581,6 @@ def runCommand(args):
|
|||
sys.stdout.write("Running test {0} ({1} {2}) - ".format(test.fullName, flavor, device));
|
||||
if args.dry_run:
|
||||
print "[SKIPPED] (dry-run)"
|
||||
continue
|
||||
# in verbose mode, terminate the line, since there will be a lot of output
|
||||
if args.verbose:
|
||||
sys.stdout.write("\n");
|
||||
|
|
@ -629,7 +640,7 @@ defaultRunDir=os.path.join(tmpDir, "cntk-test-{0}.{1}".format(time.strftime("%Y%
|
|||
runSubparser.add_argument("-r", "--run-dir", default=defaultRunDir, help="directory where to store test output, default: a random dir within /tmp")
|
||||
runSubparser.add_argument("--update-baseline", action='store_true', help="update baseline file(s) instead of matching them")
|
||||
runSubparser.add_argument("-v", "--verbose", action='store_true', help="verbose output - dump all output of test script")
|
||||
runSubparser.add_argument("-n", "--dry-run", action='store_true', help="do not run the tests, only print test names and condfigurations to be run")
|
||||
runSubparser.add_argument("-n", "--dry-run", action='store_true', help="do not run the tests, only print test names and configurations to be run along with full command lines")
|
||||
|
||||
runSubparser.set_defaults(func=runCommand)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
#!/bin/bash
|
||||
|
||||
BinaryPath=$TEST_CNTK_BINARY
|
||||
|
||||
if [ "$TEST_DEVICE" == "cpu" ]; then
|
||||
CNTKDeviceId=-1
|
||||
elif [ "$TEST_DEVICE" == "gpu" ]; then
|
||||
CNTKDeviceId=0
|
||||
else
|
||||
echo "Error: Unknown TEST_DEVICE specified!"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
LogFileName=
|
||||
|
||||
ConfigDir=$TEST_DIR
|
||||
RunDir=$TEST_RUN_DIR
|
||||
DataDir=$TEST_DATA_DIR
|
||||
|
||||
MPIMode=0
|
||||
MPIArgs=
|
||||
|
||||
DeleteExistingModels=1
|
||||
|
||||
# Helper function to print and run a command
|
||||
run()
|
||||
{
|
||||
cmd=$1
|
||||
shift
|
||||
if [ "$DRY_RUN" == "1" ]; then
|
||||
workingDir=$PWD
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
cmd=$(cygpath -aw $cmd)
|
||||
workingDir=$(cygpath -aw $workingDir)
|
||||
fi
|
||||
echo Working Directory: $workingDir
|
||||
echo Full command: "$cmd" "$@"
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
if [[ $MPIMode == 0 ]]; then
|
||||
TEST_ROOT_DIR_ESCAPED=`echo -n $(cygpath -aw $TEST_ROOT_DIR) | sed 's/\\\\/\\\\\\\\/g'`
|
||||
echo VS debugging command args: "$@" | sed "s/$TEST_ROOT_DIR_ESCAPED/\\$\\(SolutionDir\\)\\\\Tests/g"
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
else
|
||||
echo === Running "$cmd" "$@"
|
||||
"$cmd" "$@"
|
||||
return $?
|
||||
fi
|
||||
}
|
||||
|
||||
# Function for launching the CNTK executable
|
||||
# cntkrun <CNTK config file name> <additional CNTK args>
|
||||
cntkrun()
|
||||
{
|
||||
configFileName=$1
|
||||
additionalCNTKArgs=$2
|
||||
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
# When running on cygwin translating /cygdrive/xxx paths to proper windows paths:
|
||||
ConfigDir=$(cygpath -aw $ConfigDir)
|
||||
RunDir=$(cygpath -aw $RunDir)
|
||||
DataDir=$(cygpath -aw $DataDir)
|
||||
fi
|
||||
|
||||
CNTKArgs="configFile=$ConfigDir/$configFileName RunDir=$RunDir DataDir=$DataDir DeviceId=$CNTKDeviceId $additionalCNTKArgs"
|
||||
if [ "$LogFileName" != "" ]; then
|
||||
CNTKArgs="$CNTKArgs stderr=$RunDir/$LogFileName"
|
||||
fi
|
||||
|
||||
modelsDir=$TEST_RUN_DIR/models
|
||||
if [[ $DeleteExistingModels == 1 ]]; then
|
||||
[ -d $modelsDir ] && rm -rf $modelsDir
|
||||
fi
|
||||
mkdir -p $modelsDir || exit $?
|
||||
|
||||
if [[ $MPIMode == 0 ]]; then
|
||||
run "$BinaryPath" $CNTKArgs
|
||||
else
|
||||
run "$MPI_BINARY" $MPIArgs $BinaryPath $CNTKArgs
|
||||
fi
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
# Function for launching a parallel CNTK run with MPI
|
||||
# cntkmpirun <MPI args> <CNTK config file name> <additional CNTK args>
|
||||
cntkmpirun()
|
||||
{
|
||||
# Since we use the MS MPI program on Windows, the CNTK binary path argument
|
||||
# passed to mpiexec must be in the windows format
|
||||
if [ "$OS" == "Windows_NT" ]; then
|
||||
BinaryPath=$(cygpath -aw $BinaryPath)
|
||||
fi
|
||||
|
||||
MPIMode=1
|
||||
MPIArgs=$1
|
||||
|
||||
cntkrun "$2" "$3"
|
||||
return $?
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче