Add profiling error info (#91)

.github/workflows/integration-test.yml

@@ -14,10 +14,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6.10
+        python-version: 3.7
 
     - name: Cache
       uses: actions/cache@v2
@@ -32,14 +32,16 @@ jobs:
 
     - name: Install dependencies
       run: |
-        pip install tensorflow==1.15.0
+        pip install tensorflow==2.6.0
         pip install onnx==1.9.0
         pip install torch==1.9.0
         pip install torchvision==0.10.0
         pip install onnx-simplifier
         
     - name: Install nn-Meter
-      run: pip install -U .
+      run: |
+        pip install -U .
+        pip install protobuf==3.20.3
 
     - name: Integration test
       run: python tests/integration_test/test_latency_predictor.py

.github/workflows/integration-test_nni_based_torch.yml

@@ -14,10 +14,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6.10
+        python-version: 3.7
 
     - name: Cache
       uses: actions/cache@v2

.github/workflows/integration-test_onnx_based_torch.yml

@@ -14,10 +14,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6.10
+        python-version: 3.7
 
     - name: Cache
       uses: actions/cache@v2
@@ -38,7 +38,9 @@ jobs:
         pip install onnx-simplifier==0.3.6
         
     - name: Install nn-Meter
-      run: pip install -U .
+      run: |
+        pip install -U .
+        pip install protobuf==3.20.3
 
     - name: Integration test
       run: python tests/integration_test/test_latency_predictor_torch.py --apply-onnx

nn_meter/builder/backend_meta/utils.py

@@ -117,6 +117,7 @@ def dump_profiled_results(results, detail = False, metrics = ["latency"]):
 def read_profiled_results(results):
     results_copy = copy.deepcopy(results)
     for item in results_copy.values():
+        if isinstance(item, dict):
             for model in item.values():
                 if 'latency' in model:
                     model['latency'] = Latency(model['latency'])

nn_meter/builder/kernel_predictor_builder/predictor_builder/extract_feature.py

@@ -180,6 +180,10 @@ def get_data_by_profiled_results(kernel_type, feature_parser, cfgs_path, labs_pa
         except:
             pass
 
+    if len(features) == 0:
+        raise ValueError(f"Didn't find any data of {kernel_type} for predictor training. There maybe some error about model profiling. Please check the profiling error in " \
+            "`<workspace>/predictor_build/results/profile_error.log`")
+
     # save features and latency information to `save_path`
     if save_path:
        import pandas as pd

nn_meter/ir_converter/frozenpb_converter/shape_inference.py

@@ -52,7 +52,7 @@ class ShapeInference:
         "Relu6",
         "Selu",
         "LeakyReLU",
-        "Elu"
+        "Elu",
         "Softmax",
 
         "NoOp"

tests/integration_test/test_latency_predictor.py

@@ -79,15 +79,19 @@ def integration_test(model_type, url, ppath, output_name = "tests/integration_te
     
     # start testing
     for pred_name, pred_version in get_predictors():
-        try:
+        # try:
         since = time.time()
         # print(f'nn-meter --{model_type} {ppath} --predictor {pred_name} --predictor-version {pred_version}')
         result = subprocess.check_output(['nn-meter', 'predict', f'--{model_type}', f'{ppath}', '--predictor', f'{pred_name}', '--predictor-version', f'{pred_version}'])
         runtime = time.time() - since
-        except NotImplementedError:
-            logging.error(f"Meets ERROR when checking --{model_type} {ppath} --predictor {pred_name} --predictor-version {pred_version}")
+        # except NotImplementedError:
+        #     logging.error(f"Meets ERROR when checking --{model_type} {ppath} --predictor {pred_name} --predictor-version {pred_version}")
 
         latency_list = parse_latency_info(result.decode('utf-8'))
+        print(model_type)
+        print(latency_list)
+        print("-----")
+        os.system("cat tests/integration_test/test_result.txt")
         for model, latency in latency_list:
             item = f'{model}, {model_type}, {pred_name}, {pred_version}, {round(float(latency), 4)}\n'
             # print(item)