Add flake8 & Fix coding style

.flake8

@@ -0,0 +1,13 @@
+[flake8]
+ignore =
+    E741,
+    W503
+exclude =
+    .git,
+    .idea,
+    __pycache__/,
+    env/,
+    venv/,
+    .vscode/
+max-line-length = 120
+per-file-ignores = __init__.py:F401

demo.py

@@ -16,7 +16,7 @@ def test_pytorch_models(args, predictor):
     inception = models.inception_v3()
     googlenet = models.googlenet()
     shufflenet = models.shufflenet_v2_x1_0()
-    mobilenet_v2 = models.mobilenet_v2()
+    mobilenet_v2 = models.mobilenet_v2()  # noqa: F841
     resnext50_32x4d = models.resnext50_32x4d()
     wide_resnet50_2 = models.wide_resnet50_2()
     mnasnet = models.mnasnet1_0()
@@ -56,7 +56,6 @@ def test_pb_models(args, predictor):
     for model in models:
         latency = predictor.predict(model)
         print(model.split("/")[-1], latency)
-       # break
 
 
 if __name__ == "__main__":
@@ -87,7 +86,7 @@ if __name__ == "__main__":
             latency = predictor.predict(args.input_model)
             # test_onnx_models(args,predictor)
             # test_pb_models(args,predictor)
-            #test_pytorch_models(args, predictor)
+            # test_pytorch_models(args, predictor)
             print('predict latency', latency)
         else:
             raise NotImplementedError

nn_meter/ir_converters/frozenpb_converter/frozenpb_converter.py

@@ -1,7 +1,5 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
-import sys
-import os
 import numpy as np
 
 from nn_meter.utils.graphe_tool import Graphe

nn_meter/ir_converters/frozenpb_converter/frozenpb_parser.py

@@ -2,9 +2,6 @@
 # Licensed under the MIT license.
 from .protobuf_helper import ProtobufHelper
 from .shape_fetcher import ShapeFetcher
-from tensorflow import io
-from tensorflow import gfile
-from google.protobuf import text_format
 import tensorflow as tf
 import copy
 import re
@@ -90,7 +87,6 @@ class FrozenPbParser:
                     if graph[graph_node]["attr"]["type"] == "Split" and ":" not in graph_node:
                         logging.info("Find split main node %s." % graph_node)
                         split_node_name = graph_node
-                        split_node_child = []
                         for node_name in graph.keys():
                             idx = re.findall(
                                 r"%s:(\d+)" %
@@ -196,7 +192,10 @@ class FrozenPbParser:
                     if len(node_attr) > 0:
                         logging.info("Find regex matching node %s" % node.name)
                         for attr_name in target_node.attr.keys():
-                            if attr_name == "value" and "weight" not in node.name and "BatchNorm" not in node.name and "kernel" not in node.name:
+                            if (attr_name == "value"
+                                    and "weight" not in node.name
+                                    and "BatchNorm" not in node.name
+                                    and "kernel" not in node.name):
                                 node_attr_name = attr_as_node[node.op][
                                     "attr_name"]
                                 if node_attr_name not in attr_dict.keys():
@@ -212,7 +211,10 @@ class FrozenPbParser:
                     if target_node.name == attr_as_node[node.op]["node_name"](
                             node.name):
                         for attr_name in target_node.attr.keys():
-                            if attr_name == "value" and "weight" not in node.name and "BatchNorm" not in node.name and "kernel" not in node.name:
+                            if (attr_name == "value"
+                                    and "weight" not in node.name
+                                    and "BatchNorm" not in node.name
+                                    and "kernel" not in node.name):
                                 attr_dict[attr_as_node[node.op]["attr_name"]] = copy.deepcopy(
                                     attr_as_node[node.op]["node_value"](target_node.attr[attr_name].tensor)
                                 )

nn_meter/ir_converters/frozenpb_converter/shape_inference.py

@@ -98,15 +98,15 @@ class ShapeInference:
             pad_size = [pad_top, pad_bottom, pad_left, pad_right]
         elif padding == "VALID":
             outh = math.ceil(
-                (ph.get_h(input_shape) -
-                 ph.get_h(k_size) +
-                    1) /
-                ph.get_h(strides))
+                (ph.get_h(input_shape)
+                 - ph.get_h(k_size)
+                 + 1)
+                / ph.get_h(strides))
             outw = math.ceil(
-                (ph.get_h(input_shape) -
-                 ph.get_h(k_size) +
-                    1) /
-                ph.get_w(strides))
+                (ph.get_h(input_shape)
+                 - ph.get_h(k_size)
+                 + 1)
+                / ph.get_w(strides))
 
             pad_size = [0, 0, 0, 0]
         else:
@@ -452,7 +452,6 @@ class ShapeInference:
         )
 
         k_size = weight_shape[:2]
-        cin = weight_shape[2]
         cout = weight_shape[3]
 
         if node["attr"]["attr"]["strides"][::3] != [1, 1]:

nn_meter/ir_converters/onnx_converter/converter.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT license.
 import networkx as nx
 from .utils import get_tensor_shape
-from .constants import *
+from .constants import SLICE_TYPE
 from itertools import chain
 import logging
 from onnx import AttributeProto, shape_inference

nn_meter/ir_converters/onnx_converter/utils.py

@@ -1,6 +1,5 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
-from .constants import *
 
 
 def get_tensor_shape(tensor):

nn_meter/ir_converters/torch_converter/converter.py

@@ -8,7 +8,9 @@ class TorchConverter(OnnxConverter):
     def __init__(self, model, args):
         """
         @params
-        args: model input, refer to https://pytorch.org/docs/stable/onnx.html#example-end-to-end-alexnet-from-pytorch-to-onnx for more information.
+        args: model input, refer to
+        https://pytorch.org/docs/stable/onnx.html#example-end-to-end-alexnet-from-pytorch-to-onnx
+        for more information.
         """
         with tempfile.TemporaryFile() as fp:
             torch.onnx.export(model, args, fp)

nn_meter/kerneldetection/detection/detector.py

@@ -3,7 +3,7 @@
 from nn_meter.kerneldetection.rulelib.rule_reader import RuleReader
 from nn_meter.kerneldetection.rulelib.rule_splitter import RuleSplitter
 from nn_meter.utils.graphe_tool import Graphe
-from nn_meter.kerneldetection.utils.constants import *
+from nn_meter.kerneldetection.utils.constants import DUMMY_TYPES
 from nn_meter.kerneldetection.utils.ir_tools import convert_nodes
 
 

nn_meter/kerneldetection/rulelib/rule_reader.py

@@ -32,6 +32,9 @@ class RuleReader:
             return self.rules[rule]["obey"]
 
     def _extract_fusible(self):
+        def get_name(i):
+            return f"{ops[i]}_{i}"
+
         self.fusible = []
         self.fusion_units = {}
         for name, rule in self.rules.items():
@@ -41,7 +44,6 @@ class RuleReader:
                     self.fusible.append((ops[0], ops[1]))
                 elif len(ops) > 2:
                     fusion_unit = {}
-                    def get_name(i): return f"{ops[i]}_{i}"
                     for i in range(0, len(ops)):
                         fusion_unit[get_name(i)] = {
                             "attr": {"type": ops[i],

nn_meter/kerneldetection/utils/ir_tools.py

@@ -1,7 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 import copy
-from .constants import *
+from .constants import OP_ALIAS
 
 
 def convert_nodes(graph):

nn_meter/nn_meter.py

@@ -35,11 +35,9 @@ class nnMeter:
         """
         if isinstance(model, str):
             graph = model_file_to_graph(model, model_type)
-            modelname = model.split("/")[-1].split(".")[0]
         else:
             graph = model_to_graph(model, model_type, input_shape=input_shape)
         self.kd.load_graph(graph)
 
-       # kernel_result = {modelname: self.kd.kernels}
         py = nn_predict(self.kernel_predictors, self.kd.kernels)
         return py

nn_meter/prediction/load_predictors.py

@@ -1,7 +1,6 @@
 import pickle
 import os
 from glob import glob
-import shutil
 from zipfile import ZipFile
 from tqdm import tqdm
 import requests
@@ -27,17 +26,17 @@ def loading_to_local(configs, hardware, dir="data/predictorzoo"):
             predictors[pname] = model
     fusionrule = ppath + "/rule_" + hardware + ".json"
     print(fusionrule)
-    if os.path.isfile(fusionrule) == False:
+    if not os.path.isfile(fusionrule):
         raise ValueError(
-            "check your fusion rule path, file " +
-            fusionrule +
-            " does not exist！")
+            "check your fusion rule path, file "
+            + fusionrule
+            + " does not exist！")
     return predictors, fusionrule
 
 
 def download_from_url(urladdr, ppath, filename):
     file_name = ppath + "/" + ".zip"
-    if os.path.isdir(ppath) == False:
+    if not os.path.isdir(ppath):
         os.makedirs(ppath)
 
     print("download from " + urladdr)

nn_meter/prediction/predictors/extract_feature.py

@@ -2,13 +2,10 @@
 # Licensed under the MIT license.
 import numpy as np
 from sklearn.metrics import mean_squared_error
-import shutil
-import json
 
 
 def get_accuracy(y_pred, y_true, threshold=0.01):
     a = (y_true - y_pred) / y_true
-    c = abs(y_true - y_pred)
     b = np.where(abs(a) <= threshold)
     return len(b[0]) / len(y_true)
 

nn_meter/prediction/predictors/predict_by_kernel.py

@@ -1,8 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
-import sys
-from .utils import *
-from .extract_feature import *
+from .utils import get_kernel_name
+from .extract_feature import get_predict_features
 
 
 # to speed up, we merge conv and dwconv related kernels into

nn_meter/prediction/predictors/utils.py

@@ -1,7 +1,5 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
-import pickle
-import os
 
 # multiple similar kernels share one kernel predictor, since the latency
 # difference is negligible

nn_meter/utils/graphe_tool.py

@@ -298,10 +298,6 @@ class Graphe:
                         sub_graph.get_graph()[
                             sub_fetch_graph[op_entry]]["inbounds"])
                     dummy_op.attr["T"].type = 1
-                    # if 'graph_head' in sub_graph.get_graph()[sub_fetch_graph[op_entry]]['attr']['attr']:
-                    #     dummy_op.attr['shape'] = []
-                    #     dummy_op.attr['shape'].dim = list(map(int, sub_graph.get_graph()[sub_fetch_graph[op_entry]]['attr']['attr']['graph_head'].split(',')))
-                    # print(dummy_op)
                 else:
                     # --- Fetch the main op ---
                     node = copy.deepcopy(self.graph[op_entry]["attr"]["node"])

requirements.txt

@@ -10,3 +10,4 @@ protobuf==3.17.1
 PyYAML==5.4.1
 scikit_learn==0.24.2
 torch==1.8.1
+flake8==3.9.2