microsoft
/
MMdnn
зеркало из https://github.com/microsoft/MMdnn.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Seperate tests into files.

This commit is contained in:
kitstar 2018-08-28 01:05:06 +00:00
Родитель dde0b4a757
Коммит 71298b5959
17 изменённых файлов: 398 добавлений и 199 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

93
.travis.yml
Просмотреть файл

@ -1,13 +1,51 @@
sudo: required
dist: xenial
os:
- linux
- osx
language: python
python:
- "2.7"
- "3.6"
env:
- TEST_PARSER=tensorflow TEST_ONNX=true
- TEST_PARSER=tensorflow TEST_ONNX=false
- TEST_PARSER=keras TEST_ONNX=true
- TEST_PARSER=keras TEST_ONNX=false
- TEST_PARSER=cntk TEST_ONNX=true
- TEST_PARSER=cntk TEST_ONNX=false
- TEST_PARSER=caffe TEST_ONNX=true
- TEST_PARSER=caffe TEST_ONNX=false
- TEST_PARSER=mxnet TEST_ONNX=true
- TEST_PARSER=mxnet TEST_ONNX=false
- TEST_PARSER=pytorch TEST_ONNX=true
- TEST_PARSER=pytorch TEST_ONNX=false
- TEST_PARSER=paddle TEST_ONNX=true
- TEST_PARSER=paddle TEST_ONNX=false
- TEST_PARSER=coreml TEST_ONNX=true
- TEST_PARSER=coreml TEST_ONNX=false
- TEST_PARSER=darknet TEST_ONNX=true
- TEST_PARSER=darknet TEST_ONNX=false
- TEST_PARSER=tensorflow_frozen TEST_ONNX=true
- TEST_PARSER=tensorflow_frozen TEST_ONNX=false
cache:
directories:
- $HOME/.cache/pip
- $TRAVIS_BUILD_DIR/tests/cache
addons:
apt:
update: true
before_install:
- export PATH=$PATH
- sudo apt-get install -y openmpi-bin
install:
- pip install -q -r $(python requirements/select_requirements.py)
before_script: true
@ -17,53 +55,14 @@ after_success: true
after_script: true
script: bash test.sh
script: bash test.sh $TEST_PARSER
matrix:
fast_finish: true
allow_failures:
- os: osx
notifications:
email:
on_success: never
on_failure: never
matrix:
include:
- name: "MMdnn no-onnx test with py2/ubuntu 16.04"
language: python
python: "2.7"
env: TEST_ONNX=false
install:
- pip install -q -r requirements.txt
- pip install -q http://download.pytorch.org/whl/cpu/torch-0.4.0-cp27-cp27mu-linux_x86_64.whl
- pip install -q torchvision
# - pip install -q paddlepaddle==0.11.0
- name: "MMdnn onnx test with py2/ubuntu 16.04"
language: python
python: "2.7"
env: TEST_ONNX=true
install:
- pip install -q -r requirements.txt
- pip install -q http://download.pytorch.org/whl/cpu/torch-0.4.0-cp27-cp27mu-linux_x86_64.whl
- pip install -q torchvision
# - pip install -q paddlepaddle==0.11.0
- pip install -q onnx==1.2.1
- pip install -q onnx-tf==1.1.2
- name: "MMdnn no-onnx test with py3/ubuntu 16.04"
language: python
python: "3.6"
env: TEST_ONNX=false
install:
- pip install -q -r requirements.txt
- pip install -q http://download.pytorch.org/whl/cpu/torch-0.4.0-cp36-cp36m-linux_x86_64.whl
- pip install -q torchvision
- name: "MMdnn onnx test with py3/ubuntu 16.04"
language: python
python: "3.6"
env: TEST_ONNX=true
install:
- pip install -q -r requirements.txt
- pip install -q http://download.pytorch.org/whl/cpu/torch-0.4.0-cp36-cp36m-linux_x86_64.whl
- pip install -q torchvision
- pip install -q onnx==1.2.1
- pip install -q onnx-tf==1.1.2

15
requirements/linux-py2.txt Normal file
Просмотреть файл

@ -0,0 +1,15 @@
paddlepaddle==0.11.0
six==1.10.0
numpy==1.14.5
protobuf==3.6.1
pillow == 5.2.0
setuptools==39.1.0
tensorflow==1.10.0
keras==2.1.6
coremltools==0.8
mxnet==1.1.0.post0
cntk==2.5.1
http://download.pytorch.org/whl/cpu/torch-0.4.0-cp27-cp27mu-linux_x86_64.whl
torchvision
onnx==1.2.1
onnx-tf==1.1.2

6
requirements.txt → requirements/linux-py3.txt
Просмотреть файл

@ -7,4 +7,8 @@ tensorflow==1.10.0
keras==2.1.6
coremltools==0.8
mxnet==1.1.0.post0
cntk==2.5
cntk==2.5.1
http://download.pytorch.org/whl/cpu/torch-0.4.0-cp36-cp36m-linux_x86_64.whl
torchvision
onnx==1.2.1
onnx-tf==1.1.2

56
requirements/select_requirements.py Normal file
Просмотреть файл

@ -0,0 +1,56 @@
#!/usr/bin/python
"""
To have a single pip command that uses the specific requirements file use this
in a shell script for posix OS::
pip install -r $(select_requirements.py)
On windows, create a bat of cmd file that loads the windows-specific
requirements directly::
for /f %%i in ('python select_requirements.py') do (set req_file="%%i")
pip install -r %req_file%
"""
from __future__ import print_function
import os
import platform
import struct
import sys
# major python major_python_versions as python2 and python3
major_python_versions = tuple(map(str, platform.python_version_tuple()))
python2 = major_python_versions[0] == '2'
python3 = major_python_versions[0] == '3'
# operating system
sys_platform = str(sys.platform).lower()
linux = 'linux' in sys_platform
windows = 'win32' in sys_platform
cygwin = 'cygwin' in sys_platform
solaris = 'sunos' in sys_platform
macosx = 'darwin' in sys_platform
posix = 'posix' in os.name.lower()
def select_requirements_file():
"""
Print the path to a requirements file based on some os/arch condition.
"""
if windows:
print('requirements/win.txt')
elif macosx:
print('requirements/mac.txt')
elif linux:
if python2:
print('requirements/linux-py2.txt')
elif python3:
print('requirements/linux-py3.txt')
elif cygwin:
print('requirements/cygwin.txt')
else:
raise Exception('Unsupported OS/platform')
if __name__ == '__main__':
select_requirements_file()

2
test.sh
Просмотреть файл

@ -27,7 +27,7 @@ bash -c "while true; do echo \$(date) - building ...; sleep $PING_SLEEP; done" &
PING_LOOP_PID=$!
# My build is using maven, but you could build anything with this, E.g.
python -m pytest -s -q tests/ >> $BUILD_OUTPUT 2>&1
python -m pytest -s -q tests/test_$1.py >> $BUILD_OUTPUT 2>&1
# The build finished without returning an error so dump a tail of the output
dump_output

9
tests/test_caffe.py Normal file
Просмотреть файл

@ -0,0 +1,9 @@
from __future__ import absolute_import
from __future__ import print_function
from test_conversion_imagenet import TestModels
def test_caffe():
tester = TestModels()
tester._test_function('caffe', tester.CaffeParse)

13
tests/test_cntk.py Normal file
Просмотреть файл

@ -0,0 +1,13 @@
from __future__ import absolute_import
from __future__ import print_function
import sys
from test_conversion_imagenet import TestModels
def test_cntk():
try:
tester = TestModels()
tester._test_function('cntk', tester.CntkParse)
except ImportError:
print('Please install cntk! Or cntk is not supported in your platform.', file=sys.stderr)

230
tests/test_conversion_imagenet.py
Просмотреть файл

@ -5,90 +5,11 @@ import os
TEST_ONNX = os.environ.get('TEST_ONNX')
import sys
import imp
import unittest
import numpy as np
from mmdnn.conversion.examples.imagenet_test import TestKit
def _compute_SNR(x,y):
noise = x - y
noise_var = np.sum(noise ** 2) / len(noise) + 1e-7
signal_energy = np.sum(y ** 2) / len(y)
max_signal_energy = np.amax(y ** 2)
SNR = 10 * np.log10(signal_energy / noise_var)
PSNR = 10 * np.log10(max_signal_energy / noise_var)
return SNR, PSNR
def _compute_max_relative_error(x, y):
from six.moves import xrange
rerror = 0
index = 0
for i in xrange(len(x)):
den = max(1.0, np.abs(x[i]), np.abs(y[i]))
if np.abs(x[i]/den - y[i] / den) > rerror:
rerror = np.abs(x[i] / den - y[i] / den)
index = i
return rerror, index
def _compute_L1_error(x, y):
return np.linalg.norm(x - y, ord=1)
def ensure_dir(f):
d = os.path.dirname(f)
if not os.path.exists(d):
os.makedirs(d)
def checkfrozen(f):
if f == 'tensorflow_frozen':
return 'tensorflow'
else:
return f
class CorrectnessTest(unittest.TestCase):
err_thresh = 0.15
snr_thresh = 12
psnr_thresh = 30
@classmethod
def setUpClass(cls):
""" Set up the unit test by loading common utilities.
"""
pass
def _compare_outputs(self, original_framework, target_framework, network_name, original_predict, converted_predict, need_assert=True):
# Function self.assertEquals has deprecated, change to assertEqual
if (converted_predict is None or original_predict is None) and not need_assert:
return
# self.assertEqual(original_predict.shape, converted_predict.shape)
original_predict = original_predict.flatten()
converted_predict = converted_predict.flatten()
len1 = original_predict.shape[0]
len2 = converted_predict.shape[0]
length = min(len1, len2)
original_predict = np.sort(original_predict)[::-1]
converted_predict = np.sort(converted_predict)[::-1]
original_predict = original_predict[0:length]
converted_predict = converted_predict[0:length]
error, ind = _compute_max_relative_error(converted_predict, original_predict)
L1_error = _compute_L1_error(converted_predict, original_predict)
SNR, PSNR = _compute_SNR(converted_predict, original_predict)
print("error:", error)
print("L1 error:", L1_error)
print("SNR:", SNR)
print("PSNR:", PSNR)
if need_assert:
self.assertGreater(SNR, self.snr_thresh, "Error in converting {} from {} to {}".format(network_name, original_framework, target_framework))
self.assertGreater(PSNR, self.psnr_thresh, "Error in converting {} from {} to {}".format(network_name, original_framework, target_framework))
self.assertLess(error, self.err_thresh, "Error in converting {} from {} to {}".format(network_name, original_framework, target_framework))
import utils
from utils import *
class TestModels(CorrectnessTest):
@ -97,6 +18,11 @@ class TestModels(CorrectnessTest):
cachedir = "tests/cache/"
tmpdir = "tests/tmp/"
def __init__(self, test_table=None, methodName='test_nothing'):
super(TestModels, self).__init__(methodName)
@staticmethod
def TensorFlowParse(architecture_name, image_path):
from mmdnn.conversion.examples.tensorflow.extractor import tensorflow_extractor
@ -539,43 +465,48 @@ class TestModels(CorrectnessTest):
@staticmethod
def CaffeEmit(original_framework, architecture_name, architecture_path, weight_path, image_path):
import caffe
from mmdnn.conversion.caffe.caffe_emitter import CaffeEmitter
try:
import caffe
from mmdnn.conversion.caffe.caffe_emitter import CaffeEmitter
# IR to code
converted_file = original_framework + '_caffe_' + architecture_name + "_converted"
converted_file = converted_file.replace('.', '_')
emitter = CaffeEmitter((architecture_path, weight_path))
emitter.run(converted_file + '.py', converted_file + '.npy', 'test')
del emitter
del CaffeEmitter
# IR to code
converted_file = original_framework + '_caffe_' + architecture_name + "_converted"
converted_file = converted_file.replace('.', '_')
emitter = CaffeEmitter((architecture_path, weight_path))
emitter.run(converted_file + '.py', converted_file + '.npy', 'test')
del emitter
del CaffeEmitter
# import converted model
imported = imp.load_source('CaffeModel', converted_file + '.py')
# import converted model
imported = imp.load_source('CaffeModel', converted_file + '.py')
imported.make_net(converted_file + '.prototxt')
imported.gen_weight(converted_file + '.npy', converted_file + '.caffemodel', converted_file + '.prototxt')
model_converted = caffe.Net(converted_file + '.prototxt', converted_file + '.caffemodel', caffe.TEST)
imported.make_net(converted_file + '.prototxt')
imported.gen_weight(converted_file + '.npy', converted_file + '.caffemodel', converted_file + '.prototxt')
model_converted = caffe.Net(converted_file + '.prototxt', converted_file + '.caffemodel', caffe.TEST)
original_framework = checkfrozen(original_framework)
func = TestKit.preprocess_func[original_framework][architecture_name]
img = func(image_path)
img = np.transpose(img, [2, 0, 1])
input_data = np.expand_dims(img, 0)
original_framework = checkfrozen(original_framework)
func = TestKit.preprocess_func[original_framework][architecture_name]
img = func(image_path)
img = np.transpose(img, [2, 0, 1])
input_data = np.expand_dims(img, 0)
model_converted.blobs[model_converted._layer_names[0]].data[...] = input_data
predict = model_converted.forward()[model_converted._blob_names[-1]][0]
converted_predict = np.squeeze(predict)
model_converted.blobs[model_converted._layer_names[0]].data[...] = input_data
predict = model_converted.forward()[model_converted._layer_names[-1]][0]
converted_predict = np.squeeze(predict)
del model_converted
del sys.modules['CaffeModel']
del caffe
os.remove(converted_file + '.py')
os.remove(converted_file + '.npy')
os.remove(converted_file + '.prototxt')
os.remove(converted_file + '.caffemodel')
del model_converted
del sys.modules['CaffeModel']
del caffe
os.remove(converted_file + '.py')
os.remove(converted_file + '.npy')
os.remove(converted_file + '.prototxt')
os.remove(converted_file + '.caffemodel')
return converted_predict
return converted_predict
except ImportError:
print ("Cannot import Caffe. Caffe Emit is not tested.")
return None
@staticmethod
@ -704,9 +635,6 @@ class TestModels(CorrectnessTest):
except ImportError:
print('Please install Onnx! Or Onnx is not supported in your platform.', file=sys.stderr)
except:
raise ValueError
finally:
del prepare
del model_converted
@ -783,23 +711,23 @@ class TestModels(CorrectnessTest):
'tensorflow' : {
'vgg19' : [OnnxEmit],
# 'inception_v1' : [OnnxEmit],
# 'inception_v3' : [OnnxEmit],
# # 'resnet_v1_50' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# # 'resnet_v1_152' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# # 'resnet_v2_50' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# # 'resnet_v2_152' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# 'mobilenet_v1_1.0' : [OnnxEmit],
# 'mobilenet_v2_1.0_224' : [OnnxEmit],
# # 'nasnet-a_large' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# 'inception_resnet_v2' : [OnnxEmit],
'inception_v1' : [OnnxEmit],
'inception_v3' : [OnnxEmit],
# 'resnet_v1_50' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# 'resnet_v1_152' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# 'resnet_v2_50' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
# 'resnet_v2_152' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
'mobilenet_v1_1.0' : [OnnxEmit],
'mobilenet_v2_1.0_224' : [OnnxEmit],
# 'nasnet-a_large' : [OnnxEmit], # POOL: strides > window_shape not supported due to inconsistency between CPU and GPU implementations
'inception_resnet_v2' : [OnnxEmit],
},
# 'tensorflow_frozen' : {
# 'inception_v1' : [OnnxEmit],
# 'inception_v3' : [OnnxEmit],
# 'mobilenet_v1_1.0' : [OnnxEmit],
# },
'tensorflow_frozen' : {
'inception_v1' : [OnnxEmit],
'inception_v3' : [OnnxEmit],
'mobilenet_v1_1.0' : [OnnxEmit],
},
'coreml' : {
'inception_v3' : [OnnxEmit],
@ -813,12 +741,12 @@ class TestModels(CorrectnessTest):
},
'paddle' : {
'resnet50' : [CaffeEmit], #crash due to gflags_reporting.cc
'vgg16' : [TensorflowEmit], # First 1000 exactly the same, the last one is different
'resnet50' : [OnnxEmit],
'vgg16' : [OnnxEmit], # First 1000 exactly the same, the last one is different
},
'pytorch' : {
# TODO: coredump
},
@ -868,23 +796,23 @@ class TestModels(CorrectnessTest):
},
'tensorflow' : {
'vgg19' : [CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit],
# 'inception_v1' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
# 'inception_v3' : [CaffeEmit, CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit],
# 'resnet_v1_50' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
# 'resnet_v1_152' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
# 'resnet_v2_50' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
# 'resnet_v2_152' : [CaffeEmit, CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit],
# 'mobilenet_v1_1.0' : [CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CaffeEmit(Crash)
# 'mobilenet_v2_1.0_224' : [CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CaffeEmit(Crash)
# 'nasnet-a_large' : [MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: KerasEmit(Slice Layer: https://blog.csdn.net/lujiandong1/article/details/54936185)
# 'inception_resnet_v2' : [CaffeEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # CoremlEmit worked once, then always crashed
'vgg19' : [CaffeEmit, CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit],
'inception_v1' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
'inception_v3' : [CaffeEmit, CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit],
'resnet_v1_50' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
'resnet_v1_152' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
'resnet_v2_50' : [CaffeEmit, CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CntkEmit
'resnet_v2_152' : [CaffeEmit, CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit],
'mobilenet_v1_1.0' : [CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CaffeEmit(Crash)
'mobilenet_v2_1.0_224' : [CoreMLEmit, CntkEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: CaffeEmit(Crash)
'nasnet-a_large' : [MXNetEmit, PytorchEmit, TensorflowEmit], # TODO: KerasEmit(Slice Layer: https://blog.csdn.net/lujiandong1/article/details/54936185)
'inception_resnet_v2' : [CaffeEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # CoremlEmit worked once, then always crashed
},
'tensorflow_frozen' : {
# 'inception_v1' : [TensorflowEmit, KerasEmit, MXNetEmit, CoreMLEmit], # TODO: CntkEmit
# 'inception_v3' : [TensorflowEmit, KerasEmit, MXNetEmit, CoreMLEmit], # TODO: CntkEmit
# 'mobilenet_v1_1.0' : [TensorflowEmit, KerasEmit, MXNetEmit, CoreMLEmit]
'inception_v1' : [TensorflowEmit, KerasEmit, MXNetEmit, CoreMLEmit], # TODO: CntkEmit
'inception_v3' : [TensorflowEmit, KerasEmit, MXNetEmit, CoreMLEmit], # TODO: CntkEmit
'mobilenet_v1_1.0' : [TensorflowEmit, KerasEmit, MXNetEmit, CoreMLEmit]
},
'coreml' : {
@ -901,7 +829,7 @@ class TestModels(CorrectnessTest):
},
'paddle' : {
'resnet50': [CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # CaffeEmit crash
'resnet50': [CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # CaffeEmit crash, due to gflags_reporting.cc
'resnet101': [CoreMLEmit, KerasEmit, MXNetEmit, PytorchEmit, TensorflowEmit], # CaffeEmit crash
# 'vgg16': [TensorflowEmit],
# 'alexnet': [TensorflowEmit]
@ -929,7 +857,7 @@ class TestModels(CorrectnessTest):
if macos_version() < (10, 13):
return False
if target_framework == 'Onnx':
if target_framework == 'Onnx' or target_framework == 'Caffe':
if converted_prediction is None:
return False
@ -978,6 +906,8 @@ class TestModels(CorrectnessTest):
print("Testing {} model all passed.".format(original_framework), file=sys.stderr)
def test_nothing(self):
pass
# def test_caffe(self):
# try:
@ -1029,8 +959,8 @@ class TestModels(CorrectnessTest):
# self._test_function('pytorch', self.PytorchParse)
def test_tensorflow(self):
self._test_function('tensorflow', self.TensorFlowParse)
# def test_tensorflow(self):
# self._test_function('tensorflow', self.TensorFlowParse)
# def test_tensorflow_frozen(self):

14
tests/test_coreml.py Normal file
Просмотреть файл

@ -0,0 +1,14 @@
from __future__ import absolute_import
from __future__ import print_function
import sys
from test_conversion_imagenet import TestModels
def test_coreml():
from coremltools.models.utils import macos_version
if macos_version() < (10, 13):
print('Coreml is not supported in your platform.', file=sys.stderr)
else:
tester = TestModels()
tester._test_function('coreml', tester.CoreMLParse)

13
tests/test_darknet.py Normal file
Просмотреть файл

@ -0,0 +1,13 @@
from __future__ import absolute_import
from __future__ import print_function
import sys
from test_conversion_imagenet import TestModels
def test_darknet():
try:
tester = TestModels()
tester._test_function('darknet', tester.DarknetParse)
except ImportError:
print('Please install Darknet! Or Darknet is not supported in your platform.', file=sys.stderr)

9
tests/test_keras.py Normal file
Просмотреть файл

@ -0,0 +1,9 @@
from __future__ import absolute_import
from __future__ import print_function
from test_conversion_imagenet import TestModels
def test_keras():
tester = TestModels()
tester._test_function('keras', tester.KerasParse)

9
tests/test_mxnet.py Normal file
Просмотреть файл

@ -0,0 +1,9 @@
from __future__ import absolute_import
from __future__ import print_function
from test_conversion_imagenet import TestModels
def test_mxnet():
tester = TestModels()
tester._test_function('mxnet', tester.MXNetParse)

17
tests/test_paddle.py Normal file
Просмотреть файл

@ -0,0 +1,17 @@
from __future__ import absolute_import
from __future__ import print_function
import sys
from test_conversion_imagenet import TestModels
def test_paddle():
# omit tensorflow lead to crash
import tensorflow as tf
try:
tester = TestModels()
tester._test_function('paddle', tester.PaddleParse)
except ImportError:
print('Please install Paddlepaddle! Or Paddlepaddle is not supported in your platform.', file=sys.stderr)

9
tests/test_pytorch.py Normal file
Просмотреть файл

@ -0,0 +1,9 @@
from __future__ import absolute_import
from __future__ import print_function
from test_conversion_imagenet import TestModels
def test_pytorch():
tester = TestModels()
tester._test_function('pytorch', tester.PytorchParse)

8
tests/test_tensorflow.py Normal file
Просмотреть файл

@ -0,0 +1,8 @@
from __future__ import absolute_import
from __future__ import print_function
from test_conversion_imagenet import TestModels
def test_tensorflow():
tester = TestModels()
tester._test_function('tensorflow', tester.TensorFlowParse)

8
tests/test_tensorflow_frozen.py Normal file
Просмотреть файл

@ -0,0 +1,8 @@
from __future__ import absolute_import
from __future__ import print_function
from test_conversion_imagenet import TestModels
def test_tensorflow_frozen():
tester = TestModels()
tester._test_function('tensorflow_frozen', tester.TensorFlowFrozenParse)

86
tests/utils.py Normal file
Просмотреть файл

@ -0,0 +1,86 @@
from __future__ import absolute_import
from __future__ import print_function
__all__ = ['ensure_dir', 'checkfrozen', 'CorrectnessTest']
import os
import unittest
import numpy as np
def _compute_SNR(x,y):
noise = x - y
noise_var = np.sum(noise ** 2) / len(noise) + 1e-7
signal_energy = np.sum(y ** 2) / len(y)
max_signal_energy = np.amax(y ** 2)
SNR = 10 * np.log10(signal_energy / noise_var)
PSNR = 10 * np.log10(max_signal_energy / noise_var)
return SNR, PSNR
def _compute_max_relative_error(x, y):
from six.moves import xrange
rerror = 0
index = 0
for i in xrange(len(x)):
den = max(1.0, np.abs(x[i]), np.abs(y[i]))
if np.abs(x[i]/den - y[i] / den) > rerror:
rerror = np.abs(x[i] / den - y[i] / den)
index = i
return rerror, index
def _compute_L1_error(x, y):
return np.linalg.norm(x - y, ord=1)
def ensure_dir(f):
d = os.path.dirname(f)
if not os.path.exists(d):
os.makedirs(d)
def checkfrozen(f):
if f == 'tensorflow_frozen':
return 'tensorflow'
else:
return f
class CorrectnessTest(unittest.TestCase):
err_thresh = 0.15
snr_thresh = 12
psnr_thresh = 30
@classmethod
def setUpClass(cls):
""" Set up the unit test by loading common utilities.
"""
pass
def _compare_outputs(self, original_framework, target_framework, network_name, original_predict, converted_predict, need_assert=True):
# Function self.assertEquals has deprecated, change to assertEqual
if (converted_predict is None or original_predict is None) and not need_assert:
return
# self.assertEqual(original_predict.shape, converted_predict.shape)
original_predict = original_predict.flatten()
converted_predict = converted_predict.flatten()
len1 = original_predict.shape[0]
len2 = converted_predict.shape[0]
length = min(len1, len2)
original_predict = np.sort(original_predict)[::-1]
converted_predict = np.sort(converted_predict)[::-1]
original_predict = original_predict[0:length]
converted_predict = converted_predict[0:length]
error, ind = _compute_max_relative_error(converted_predict, original_predict)
L1_error = _compute_L1_error(converted_predict, original_predict)
SNR, PSNR = _compute_SNR(converted_predict, original_predict)
print("error:", error)
print("L1 error:", L1_error)
print("SNR:", SNR)
print("PSNR:", PSNR)
if need_assert:
self.assertGreater(SNR, self.snr_thresh, "Error in converting {} from {} to {}".format(network_name, original_framework, target_framework))
self.assertGreater(PSNR, self.psnr_thresh, "Error in converting {} from {} to {}".format(network_name, original_framework, target_framework))
self.assertLess(error, self.err_thresh, "Error in converting {} from {} to {}".format(network_name, original_framework, target_framework))