microsoft
/
MMdnn
зеркало из https://github.com/microsoft/MMdnn.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Enable parallel pytest.

This commit is contained in:
Kit 2018-02-09 13:33:15 +08:00
Родитель 9c97611070
Коммит c4aba456d0
1 изменённых файлов: 65 добавлений и 52 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

117
tests/test_conversion_imagenet.py
Просмотреть файл

@ -1,4 +1,5 @@
import os
import sys
import six
import unittest
import numpy as np
@ -6,13 +7,6 @@ from six.moves import reload_module
import tensorflow as tf
from mmdnn.conversion.examples.imagenet_test import TestKit
from mmdnn.conversion.examples.keras.extractor import keras_extractor
from mmdnn.conversion.examples.mxnet.extractor import mxnet_extractor
from mmdnn.conversion.examples.caffe.extractor import caffe_extractor
from mmdnn.conversion.keras.keras2_parser import Keras2Parser
from mmdnn.conversion.mxnet.mxnet_parser import MXNetParser
from mmdnn.conversion.cntk.cntk_emitter import CntkEmitter
from mmdnn.conversion.tensorflow.tensorflow_emitter import TensorflowEmitter
from mmdnn.conversion.keras.keras2_emitter import Keras2Emitter
@ -64,7 +58,7 @@ class CorrectnessTest(unittest.TestCase):
print("PSNR:", PSNR)
# self.assertGreater(SNR, self.snr_thresh)
# self.assertGreater(PSNR, self.psnr_thresh)
# self.assertLess(error, self.err_thresh)
self.assertLess(error, self.err_thresh)
class TestModels(CorrectnessTest):
@ -80,42 +74,54 @@ class TestModels(CorrectnessTest):
# get original model prediction result
original_predict = tensorflow_extractor.inference(architecture_name, TestModels.cachedir, image_path)
del tensorflow_extractor
# original to IR
IR_file = TestModels.tmpdir + 'tensorflow_' + architecture_name + "_converted"
parser = TensorflowParser(
TestModels.cachedir + "imagenet_" + architecture_name + ".ckpt.meta",
TestModels.cachedir + "imagenet_" + architecture_name + ".ckpt",
None,
"MMdnn_Output")
parser.run(TestModels.tmpdir + architecture_name + "_converted")
parser.run(IR_file)
del parser
del TensorflowParser
del tensorflow_extractor
return original_predict
@staticmethod
def KerasParse(architecture_name, image_path):
from mmdnn.conversion.examples.keras.extractor import keras_extractor
from mmdnn.conversion.keras.keras2_parser import Keras2Parser
# get original model prediction result
original_predict = keras_extractor.inference(architecture_name, TestModels.cachedir, image_path)
# download model
model_filename = keras_extractor.download(architecture_name, TestModels.cachedir)
del keras_extractor
# original to IR
IR_file = TestModels.tmpdir + 'keras_' + architecture_name + "_converted"
parser = Keras2Parser(model_filename)
parser.run(TestModels.tmpdir + architecture_name + "_converted")
parser.run(IR_file)
del parser
del Keras2Parser
return original_predict
@staticmethod
def MXNetParse(architecture_name, image_path):
from mmdnn.conversion.examples.mxnet.extractor import mxnet_extractor
from mmdnn.conversion.mxnet.mxnet_parser import MXNetParser
# download model
architecture_file, weight_file = mxnet_extractor.download(architecture_name, TestModels.cachedir)
# get original model prediction result
original_predict = mxnet_extractor.inference(architecture_name, TestModels.cachedir, image_path)
del mxnet_extractor
# original to IR
import re
@ -124,20 +130,25 @@ class TestModels(CorrectnessTest):
prefix, epoch = weight_file.rsplit('-', 1)
model = (architecture_file, prefix, epoch, [3, 224, 224])
IR_file = TestModels.tmpdir + 'mxnet_' + architecture_name + "_converted"
parser = MXNetParser(model)
parser.run(TestModels.tmpdir + architecture_name + "_converted")
parser.run(IR_file)
del parser
del MXNetParser
return original_predict
@staticmethod
def CaffeParse(architecture_name, image_path):
from mmdnn.conversion.examples.caffe.extractor import caffe_extractor
# download model
architecture_file, weight_file = caffe_extractor.download(architecture_name, TestModels.cachedir)
# get original model prediction result
original_predict = caffe_extractor.inference(architecture_name,architecture_file, weight_file, image_path)
del caffe_extractor
# original to IR
from mmdnn.conversion.caffe.transformer import CaffeTransformer
@ -148,13 +159,14 @@ class TestModels(CorrectnessTest):
from mmdnn.conversion.caffe.writer import ModelSaver, PyWriter
prototxt = graph.as_graph_def().SerializeToString()
pb_path = TestModels.tmpdir + architecture_name + "_converted.pb"
IR_file = TestModels.tmpdir + 'caffe_' + architecture_name + "_converted"
pb_path = IR_file + '.pb'
with open(pb_path, 'wb') as of:
of.write(prototxt)
print ("IR network structure is saved as [{}].".format(pb_path))
import numpy as np
npy_path = TestModels.tmpdir + architecture_name + "_converted.npy"
npy_path = IR_file + '.npy'
with open(npy_path, 'wb') as of:
np.save(of, data)
print ("IR weights are saved as [{}].".format(npy_path))
@ -162,27 +174,27 @@ class TestModels(CorrectnessTest):
return original_predict
@staticmethod
def CntkEmit(original_framework, architecture_name, architecture_path, weight_path, image_path):
print("Testing {} from {} to CNTK.".format(architecture_name, original_framework))
# IR to code
converted_file = original_framework + '_cntk_' + architecture_name + "_converted"
converted_file = converted_file.replace('.', '_')
emitter = CntkEmitter((architecture_path, weight_path))
emitter.run("converted_model.py", None, 'test')
emitter.run(converted_file + '.py', None, 'test')
del emitter
# import converted model
import converted_model
reload_module (converted_model)
model_converted = converted_model.KitModel(TestModels.tmpdir + architecture_name + "_converted.npy")
model_converted = __import__(converted_file).KitModel(weight_path)
func = TestKit.preprocess_func[original_framework][architecture_name]
img = func(image_path)
predict = model_converted.eval({model_converted.arguments[0]:[img]})
converted_predict = np.squeeze(predict)
del model_converted
del converted_model
os.remove("converted_model.py")
del sys.modules[converted_file]
os.remove(converted_file + '.py')
return converted_predict
@ -191,14 +203,14 @@ class TestModels(CorrectnessTest):
print("Testing {} from {} to TensorFlow.".format(architecture_name, original_framework))
# IR to code
converted_file = original_framework + '_tensorflow_' + architecture_name + "_converted"
converted_file = converted_file.replace('.', '_')
emitter = TensorflowEmitter((architecture_path, weight_path))
emitter.run("converted_model.py", None, 'test')
emitter.run(converted_file + '.py', None, 'test')
del emitter
# import converted model
import converted_model
reload_module (converted_model)
model_converted = converted_model.KitModel(TestModels.tmpdir + architecture_name + "_converted.npy")
model_converted = __import__(converted_file).KitModel(weight_path)
input_tf, model_tf = model_converted
func = TestKit.preprocess_func[original_framework][architecture_name]
@ -209,8 +221,8 @@ class TestModels(CorrectnessTest):
sess.run(init)
predict = sess.run(model_tf, feed_dict = {input_tf : input_data})
del model_converted
del converted_model
os.remove("converted_model.py")
del sys.modules[converted_file]
os.remove(converted_file + '.py')
converted_predict = np.squeeze(predict)
return converted_predict
@ -221,14 +233,14 @@ class TestModels(CorrectnessTest):
print("Testing {} from {} to PyTorch.".format(architecture_name, original_framework))
# IR to code
converted_file = original_framework + '_keras_' + architecture_name + "_converted"
converted_file = converted_file.replace('.', '_')
emitter = PytorchEmitter((architecture_path, weight_path))
emitter.run("converted_model.py", "pytorch_weight.npy", 'test')
emitter.run(converted_file + '.py', converted_file + '.npy', 'test')
del emitter
# import converted model
import converted_model
reload_module (converted_model)
model_converted = converted_model.KitModel("pytorch_weight.npy")
model_converted = __import__(converted_file).KitModel(converted_file + '.npy')
model_converted.eval()
func = TestKit.preprocess_func[original_framework][architecture_name]
@ -242,10 +254,10 @@ class TestModels(CorrectnessTest):
predict = predict.data.numpy()
del model_converted
del converted_model
del sys.modules[converted_file]
del torch
os.remove("converted_model.py")
os.remove("pytorch_weight.npy")
os.remove(converted_file + '.py')
os.remove(converted_file + '.npy')
converted_predict = np.squeeze(predict)
return converted_predict
@ -255,14 +267,14 @@ class TestModels(CorrectnessTest):
print("Testing {} from {} to Keras.".format(architecture_name, original_framework))
# IR to code
converted_file = original_framework + '_keras_' + architecture_name + "_converted"
converted_file = converted_file.replace('.', '_')
emitter = Keras2Emitter((architecture_path, weight_path))
emitter.run("converted_model.py", None, 'test')
emitter.run(converted_file + '.py', None, 'test')
del emitter
# import converted model
import converted_model
reload_module (converted_model)
model_converted = converted_model.KitModel(TestModels.tmpdir + architecture_name + "_converted.npy")
model_converted = __import__(converted_file).KitModel(weight_path)
func = TestKit.preprocess_func[original_framework][architecture_name]
img = func(image_path)
@ -272,12 +284,12 @@ class TestModels(CorrectnessTest):
converted_predict = np.squeeze(predict)
del model_converted
del converted_model
del sys.modules[converted_file]
import keras.backend as K
K.clear_session()
os.remove("converted_model.py")
os.remove(converted_file + '.py')
return converted_predict
@ -293,12 +305,12 @@ class TestModels(CorrectnessTest):
'nasnet' : [TensorflowEmit, KerasEmit],
},
'mxnet' : {
'vgg19' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
'imagenet1k-inception-bn' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
'imagenet1k-resnet-152' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
# 'vgg19' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
# 'imagenet1k-inception-bn' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
# 'imagenet1k-resnet-152' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
'squeezenet_v1.1' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
'imagenet1k-resnext-101-64x4d' : [CntkEmit, TensorflowEmit, PytorchEmit], # Keras is too slow
'imagenet1k-resnext-50' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
# 'imagenet1k-resnext-101-64x4d' : [CntkEmit, TensorflowEmit, PytorchEmit], # Keras is too slow
# 'imagenet1k-resnext-50' : [CntkEmit, TensorflowEmit, KerasEmit, PytorchEmit],
},
'caffe' : {
'vgg19' : [KerasEmit],
@ -309,7 +321,7 @@ class TestModels(CorrectnessTest):
},
'tensorflow' : {
'inception_v1' : [TensorflowEmit, KerasEmit, PytorchEmit], # TODO: CntkEmit
}
},
}
@ -323,23 +335,24 @@ class TestModels(CorrectnessTest):
# get original model prediction result
original_predict = parser(network_name, self.image_path)
IR_file = TestModels.tmpdir + original_framework + '_' + network_name + "_converted"
for emit in self.test_table[original_framework][network_name]:
converted_predict = emit.__func__(
original_framework,
network_name,
self.tmpdir + network_name + "_converted.pb",
self.tmpdir + network_name + "_converted.npy",
IR_file + ".pb",
IR_file + ".npy",
self.image_path)
self._compare_outputs(original_predict, converted_predict)
try:
os.remove(self.tmpdir + network_name + "_converted.json")
os.remove(IR_file + ".json")
except OSError:
pass
os.remove(self.tmpdir + network_name + "_converted.pb")
os.remove(self.tmpdir + network_name + "_converted.npy")
os.remove(IR_file + ".pb")
os.remove(IR_file + ".npy")
print("Testing {} model {} passed.".format(original_framework, network_name))
print("Testing {} model all passed.".format(original_framework))