diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b50c41a..29c92bc 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -14,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.7 - name: Cache uses: actions/cache@v2 @@ -39,7 +39,9 @@ jobs: pip install onnx-simplifier - name: Install nn-Meter - run: pip install -U . + run: | + pip install -U . + pip install protobuf==3.20.3 - name: Integration test run: python tests/integration_test/test_latency_predictor.py diff --git a/.github/workflows/integration-test_nni_based_torch.yml b/.github/workflows/integration-test_nni_based_torch.yml index 1fca67a..b563ebb 100644 --- a/.github/workflows/integration-test_nni_based_torch.yml +++ b/.github/workflows/integration-test_nni_based_torch.yml @@ -14,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.7 - name: Cache uses: actions/cache@v2 diff --git a/.github/workflows/integration-test_onnx_based_torch.yml b/.github/workflows/integration-test_onnx_based_torch.yml index 152e53e..8aeb9a5 100644 --- a/.github/workflows/integration-test_onnx_based_torch.yml +++ b/.github/workflows/integration-test_onnx_based_torch.yml @@ -14,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.7 - name: Cache uses: actions/cache@v2 @@ -38,7 +38,9 @@ jobs: pip install onnx-simplifier==0.3.6 - name: Install nn-Meter - run: pip install -U . + run: | + pip install -U . + pip install protobuf==3.20.3 - name: Integration test run: python tests/integration_test/test_latency_predictor_torch.py --apply-onnx diff --git a/tests/integration_test/test_result.txt b/tests/integration_test/test_result.txt new file mode 100644 index 0000000..092af49 --- /dev/null +++ b/tests/integration_test/test_result.txt @@ -0,0 +1,73 @@ +model_name, model_type, predictor, predictor_version, latency +alexnet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 84.7938 +densenet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 360.042 +googlenet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 177.4844 +mnasnet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 51.6435 +mobilenetv1_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 66.9799 +mobilenetv2_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 45.9511 +mobilenetv3large_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 35.451 +mobilenetv3small_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 12.5589 +proxylessnas_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 51.6328 +resnet18_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 216.1971 +resnet34_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 422.9927 +resnet50_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 452.7282 +shufflenetv2_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 21.3181 +squeezenet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 97.732 +vgg11_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 786.3514 +vgg13_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 1212.2778 +vgg16_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 1633.9499 +vgg19_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 2055.622 +alexnet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 13.1248 +densenet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 73.6573 +googlenet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 34.5082 +mnasnet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 13.7294 +mobilenetv1_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 13.9721 +mobilenetv2_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 10.1537 +mobilenetv3large_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 9.9899 +mobilenetv3small_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 4.4898 +proxylessnas_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 12.5095 +resnet18_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 39.3235 +resnet34_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 74.8891 +resnet50_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 91.7313 +shufflenetv2_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 5.4239 +squeezenet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 18.0742 +vgg11_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 109.7786 +vgg13_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 158.9096 +vgg16_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 219.2417 +vgg19_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 279.5738 +alexnet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 16.8936 +densenet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 76.046 +googlenet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 40.1676 +mnasnet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 13.9227 +mobilenetv1_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 13.9529 +mobilenetv2_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 10.2174 +mobilenetv3large_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 10.2039 +mobilenetv3small_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 4.6418 +proxylessnas_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 12.741 +resnet18_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 49.4287 +resnet34_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 94.6209 +resnet50_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 101.3199 +shufflenetv2_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 5.5792 +squeezenet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 20.588 +vgg11_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 144.965 +vgg13_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 206.4038 +vgg16_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 286.5241 +vgg19_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 366.6444 +alexnet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 12.3637 +densenet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 49.3979 +googlenet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 17.8349 +mnasnet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 41.4436 +mobilenetv1_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 18.807 +mobilenetv2_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 22.6613 +mobilenetv3large_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 36.5082 +mobilenetv3small_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 19.129 +proxylessnas_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 27.3499 +resnet18_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 21.8861 +resnet34_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 40.1006 +resnet50_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 58.9656 +shufflenetv2_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 20.8282 +squeezenet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 11.7883 +vgg11_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 91.6188 +vgg13_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 140.7542 +vgg16_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 189.4833 +vgg19_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 238.2123