test vgg model by onnx_test
This commit is contained in:
jiahangxu
Родитель
4ebdc08fd2
Коммит
be3c96ac74
|
|
@ -0,0 +1,98 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
name: Integrated Test for Torch Model Based on ONNX
|
||||
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
torch-model-test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
max-parallel: 5
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python 3.6
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.6.10
|
||||
|
||||
- name: Cache
|
||||
uses: actions/cache@v2
|
||||
id: cache
|
||||
env:
|
||||
cache-name: download-cache
|
||||
with:
|
||||
path: |
|
||||
~/.nn_meter
|
||||
/home/runner/work/nn-Meter/data/testmodels
|
||||
key: ${{hashFiles('nn_meter/configs/predictors.yaml')}}-${{hashFiles('tests/integration_test.py')}}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install onnx==1.9.0
|
||||
pip install torch==1.9.0
|
||||
pip install torchvision==0.10.0
|
||||
pip install onnx-simplifier
|
||||
|
||||
- name: Install nn-Meter
|
||||
run: pip install -U .
|
||||
|
||||
- name: Cli call
|
||||
run: |
|
||||
nn-meter lat_pred --torchvision resnet18 --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision alexnet --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision vgg16 --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision squeezenet --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision densenet161 --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision inception_v3 --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision googlenet --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision shufflenet_v2 --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mobilenet_v2 --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision resnext50_32x4d --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision wide_resnet50_2 --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mnasnet --predictor cortexA76cpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision resnet18 --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision alexnet --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision vgg16 --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision squeezenet --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision densenet161 --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision inception_v3 --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision googlenet --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision shufflenet_v2 --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mobilenet_v2 --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision resnext50_32x4d --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision wide_resnet50_2 --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mnasnet --predictor adreno630gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision resnet18 --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision alexnet --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision vgg16 --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision squeezenet --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision densenet161 --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision inception_v3 --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision googlenet --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision shufflenet_v2 --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mobilenet_v2 --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision resnext50_32x4d --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision wide_resnet50_2 --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mnasnet --predictor adreno640gpu_tflite21 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision resnet18 --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision alexnet --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision vgg16 --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision squeezenet --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision densenet161 --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision inception_v3 --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision googlenet --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision shufflenet_v2 --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mobilenet_v2 --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision resnext50_32x4d --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision wide_resnet50_2 --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
nn-meter lat_pred --torchvision mnasnet --predictor myriadvpu_openvino2019r2 --predictor-version 1.0
|
||||
|
||||
- name: Subprocess call
|
||||
run: python tests/onnx_test.py
|
||||
|
||||
- name: view test file
|
||||
run: cat tests/onnx_test.txt
|
||||
|
|
@ -115,7 +115,7 @@ if __name__ == "__main__":
|
|||
integration_test_onnx_based_torch(
|
||||
model_type='torch',
|
||||
model_list=[
|
||||
'resnet18', 'alexnet', 'vgg16', 'squeezenet', 'densenet161', 'inception_v3', 'googlenet',
|
||||
'resnet18', 'alexnet', 'squeezenet', 'densenet161', 'inception_v3', 'googlenet',
|
||||
'shufflenet_v2', 'mobilenet_v2', 'resnext50_32x4d', 'wide_resnet50_2', 'mnasnet']
|
||||
)
|
||||
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
import re
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import subprocess
|
||||
from integration_test import *
|
||||
import argparse
|
||||
|
||||
|
||||
__torchvision_model_zoo__ = {
|
||||
'resnet18': 'models.resnet18()',
|
||||
'alexnet': 'models.alexnet()',
|
||||
'vgg16': 'models.vgg16()',
|
||||
'squeezenet': 'models.squeezenet1_0()',
|
||||
'densenet161': 'models.densenet161()',
|
||||
'inception_v3': 'models.inception_v3()',
|
||||
'googlenet': 'models.googlenet()',
|
||||
'shufflenet_v2': 'models.shufflenet_v2_x1_0()',
|
||||
'mobilenet_v2': 'models.mobilenet_v2()',
|
||||
'resnext50_32x4d': 'models.resnext50_32x4d()',
|
||||
'wide_resnet50_2': 'models.wide_resnet50_2()',
|
||||
'mnasnet': 'models.mnasnet1_0()',
|
||||
}
|
||||
|
||||
|
||||
# integration test to predict model latency
|
||||
def integration_test_onnx_based_torch(model_type, model_list, output_name = "tests/onnx_test.txt"):
|
||||
"""
|
||||
download the kernel predictors from the url
|
||||
@params:
|
||||
|
||||
model_type: torch
|
||||
model_list: the torchvision model waiting for latency prediction
|
||||
output_name: a summary file to save the testing results
|
||||
"""
|
||||
# if the output is not created, create it and add a title
|
||||
if not os.path.isfile(output_name):
|
||||
with open(output_name,"w") as f:
|
||||
f.write('model_name, model_type, predictor, predictor_version, latency\n')
|
||||
|
||||
# start testing
|
||||
for pred_name, pred_version in get_predictors():
|
||||
for model in model_list:
|
||||
try:
|
||||
since = time.time()
|
||||
print(" ".join(['nn-meter', 'lat_pred', '--torchvision'] + model + ['--predictor', pred_name, '--predictor-version', pred_version]))
|
||||
result = subprocess.run(
|
||||
['nn-meter', 'lat_pred', '--torchvision'] + model + ['--predictor', pred_name, '--predictor-version', pred_version],
|
||||
stdout=subprocess.PIPE)
|
||||
print(result.stderr, result.stdout)
|
||||
|
||||
runtime = time.time() - since
|
||||
print("Run time: ", runtime)
|
||||
latency_list = parse_latency_info(result.stdout.decode('utf-8'))
|
||||
for model, latency in latency_list:
|
||||
item = f'{model}, {model_type}, {pred_name}, {pred_version}, {round(float(latency), 4)}\n'
|
||||
with open(output_name, "a") as f:
|
||||
f.write(item)
|
||||
except NotImplementedError:
|
||||
logging.error(f"Meets ERROR when checking --torchvision {model} --predictor {pred_name} --predictor-version {pred_version}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
integration_test_onnx_based_torch(
|
||||
model_type='torch',
|
||||
model_list=[
|
||||
'resnet18', 'alexnet', 'list', 'squeezenet', 'densenet161', 'inception_v3', 'googlenet',
|
||||
'shufflenet_v2', 'mobilenet_v2', 'resnext50_32x4d', 'wide_resnet50_2', 'mnasnet']
|
||||
)
|
||||
|
||||
Загрузка…
Ссылка в новой задаче