refactor test script
This commit is contained in:
jiahangxu
Родитель
c9e296b306
Коммит
b4b17aba59
|
|
@ -43,35 +43,6 @@ def loading_to_local(pred_info, dir="data/predictorzoo"):
|
|||
return predictors, fusionrule
|
||||
|
||||
|
||||
# def download_from_url(urladdr, ppath):
|
||||
# """
|
||||
# download the kernel predictors from the url
|
||||
# @params:
|
||||
|
||||
# urladdr: github release url address
|
||||
# ppath: the targeting hardware_inferenceframework name
|
||||
|
||||
# """
|
||||
# file_name = os.path.join(ppath, ".zip")
|
||||
# if not os.path.isdir(ppath):
|
||||
# os.makedirs(ppath)
|
||||
|
||||
# logging.keyinfo("download from " + urladdr)
|
||||
# response = requests.get(urladdr, stream=True)
|
||||
# total_size_in_bytes = int(response.headers.get("content-length", 0))
|
||||
# block_size = 2048 # 2 Kibibyte
|
||||
# progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True)
|
||||
# with open(file_name, "wb") as file:
|
||||
# for data in response.iter_content(block_size):
|
||||
# progress_bar.update(len(data))
|
||||
# file.write(data)
|
||||
# zipfile = ZipFile(file_name)
|
||||
# zipfile.extractall(path=ppath)
|
||||
# zipfile.close()
|
||||
# progress_bar.close()
|
||||
# os.remove(file_name)
|
||||
|
||||
|
||||
def check_predictors(ppath, kernel_predictors):
|
||||
"""
|
||||
@params:
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ def parse_latency_info(info):
|
|||
return latency
|
||||
|
||||
# integration test to predict model latency
|
||||
def integration_test(model_type, url, ppath, outcsv_name = "test.txt"):
|
||||
def integration_test(model_type, url, ppath, outcsv_name = "tests/test_result.txt"):
|
||||
"""
|
||||
download the kernel predictors from the url
|
||||
@params:
|
||||
|
|
@ -85,12 +85,12 @@ def integration_test(model_type, url, ppath, outcsv_name = "test.txt"):
|
|||
if __name__ == "__main__":
|
||||
check_package_status()
|
||||
|
||||
# # check tensorflow model
|
||||
# integration_test(
|
||||
# model_type='tensorflow',
|
||||
# url = "https://github.com/Lynazhang/nnmeter/releases/download/0.1/pb_models.zip",
|
||||
# ppath = "data/testmodels/pb",
|
||||
# )
|
||||
# check tensorflow model
|
||||
integration_test(
|
||||
model_type='tensorflow',
|
||||
url = "https://github.com/Lynazhang/nnmeter/releases/download/0.1/pb_models.zip",
|
||||
ppath = "data/testmodels/pb",
|
||||
)
|
||||
|
||||
# check onnx model
|
||||
integration_test(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
test_time, model_name, model_type, predictor, predictor_version, latency, runtime
|
||||
2021-07-23 13:30:23, alexnet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 84.793836, 6.55976939201355
|
||||
2021-07-23 13:30:29, alexnet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 13.124763, 6.332932472229004
|
||||
2021-07-23 13:30:36, alexnet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 16.893621, 6.663210391998291
|
||||
2021-07-23 13:30:46, alexnet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 12.363732, 9.746519327163696
|
||||
2021-07-23 13:30:56, densenet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 360.042041, 8.327949523925781
|
||||
2021-07-23 13:31:04, densenet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 73.657286, 7.724132776260376
|
||||
2021-07-23 13:31:12, densenet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 76.046010, 8.15092396736145
|
||||
2021-07-23 13:31:23, densenet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 49.397881, 11.679940700531006
|
||||
2021-07-23 13:31:33, googlenet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 177.484368, 7.268815279006958
|
||||
2021-07-23 13:31:39, googlenet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 34.508159, 6.771365404129028
|
||||
2021-07-23 13:31:46, googlenet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 40.167635, 6.872943639755249
|
||||
2021-07-23 13:31:56, googlenet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 17.834946, 10.02686357498169
|
||||
2021-07-23 13:32:06, mnasnet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 51.643542, 7.914725065231323
|
||||
2021-07-23 13:32:13, mnasnet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 13.729393, 7.251220464706421
|
||||
2021-07-23 13:32:20, mnasnet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 13.922719, 7.110336542129517
|
||||
2021-07-23 13:32:31, mnasnet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 41.443568, 10.195728778839111
|
||||
2021-07-23 13:32:39, mobilenetv1_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 66.979897, 6.813792705535889
|
||||
2021-07-23 13:32:45, mobilenetv1_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 13.972147, 6.0407493114471436
|
||||
2021-07-23 13:32:52, mobilenetv1_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 13.952927, 6.432619094848633
|
||||
2021-07-23 13:33:02, mobilenetv1_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 18.806958, 10.206496953964233
|
||||
2021-07-23 13:33:11, mobilenetv2_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 45.951083, 7.439181804656982
|
||||
2021-07-23 13:33:18, mobilenetv2_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 10.153712, 6.502416610717773
|
||||
2021-07-23 13:33:24, mobilenetv2_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 10.217366, 6.696244716644287
|
||||
2021-07-23 13:33:34, mobilenetv2_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 22.661314, 9.925176858901978
|
||||
2021-07-23 13:33:44, mobilenetv3large_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 35.451037, 7.9491822719573975
|
||||
2021-07-23 13:33:51, mobilenetv3large_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 9.989918, 6.632967948913574
|
||||
2021-07-23 13:33:58, mobilenetv3large_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 10.203856, 7.29175066947937
|
||||
2021-07-23 13:34:09, mobilenetv3large_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 36.508180, 10.411065340042114
|
||||
2021-07-23 13:34:19, mobilenetv3small_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 12.558943, 8.954994440078735
|
||||
2021-07-23 13:34:26, mobilenetv3small_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 4.489849, 6.771005868911743
|
||||
2021-07-23 13:34:33, mobilenetv3small_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 4.641754, 7.136524200439453
|
||||
2021-07-23 13:34:44, mobilenetv3small_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 19.128956, 10.970420360565186
|
||||
2021-07-23 13:34:54, proxylessnas_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 51.632800, 8.138278722763062
|
||||
2021-07-23 13:35:01, proxylessnas_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 12.509470, 6.533860445022583
|
||||
2021-07-23 13:35:08, proxylessnas_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 12.740986, 6.965597629547119
|
||||
2021-07-23 13:35:18, proxylessnas_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 27.349860, 10.163507223129272
|
||||
2021-07-23 13:35:27, resnet18_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 216.197146, 6.994403123855591
|
||||
2021-07-23 13:35:33, resnet18_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 39.323517, 6.598017692565918
|
||||
2021-07-23 13:35:40, resnet18_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 49.428744, 7.243346214294434
|
||||
2021-07-23 13:35:51, resnet18_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 21.886077, 10.071426153182983
|
||||
2021-07-23 13:36:00, resnet34_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 422.992665, 7.88179349899292
|
||||
2021-07-23 13:36:07, resnet34_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 74.889139, 6.485397815704346
|
||||
2021-07-23 13:36:14, resnet34_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 94.620914, 7.087299346923828
|
||||
2021-07-23 13:36:25, resnet34_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 40.100624, 11.421797513961792
|
||||
2021-07-23 13:36:35, resnet50_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 452.728200, 7.855477333068848
|
||||
2021-07-23 13:36:42, resnet50_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 91.731268, 6.86454176902771
|
||||
2021-07-23 13:36:50, resnet50_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 101.319897, 8.023729801177979
|
||||
2021-07-23 13:37:01, resnet50_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 58.965621, 11.179357290267944
|
||||
2021-07-23 13:37:11, shufflenetv2_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 21.318069, 7.999987363815308
|
||||
2021-07-23 13:37:18, shufflenetv2_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 5.423899, 7.107918739318848
|
||||
2021-07-23 13:37:26, shufflenetv2_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 5.579220, 8.052603006362915
|
||||
2021-07-23 13:37:38, shufflenetv2_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 20.828206, 11.585219860076904
|
||||
2021-07-23 13:37:47, squeezenet_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 97.731958, 7.346490144729614
|
||||
2021-07-23 13:37:53, squeezenet_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 18.074223, 5.935956239700317
|
||||
2021-07-23 13:38:00, squeezenet_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 20.588005, 7.057062149047852
|
||||
2021-07-23 13:38:10, squeezenet_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 11.788279, 9.855776309967041
|
||||
2021-07-23 13:38:19, vgg11_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 786.351386, 6.857112407684326
|
||||
2021-07-23 13:38:25, vgg11_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 109.778642, 6.241668462753296
|
||||
2021-07-23 13:38:32, vgg11_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 144.964981, 6.611438751220703
|
||||
2021-07-23 13:38:42, vgg11_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 91.618774, 9.976903676986694
|
||||
2021-07-23 13:38:51, vgg13_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 1212.277769, 7.038971185684204
|
||||
2021-07-23 13:38:58, vgg13_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 158.909608, 7.172670364379883
|
||||
2021-07-23 13:39:05, vgg13_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 206.403816, 6.7839601039886475
|
||||
2021-07-23 13:39:15, vgg13_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 140.754230, 9.951710939407349
|
||||
2021-07-23 13:39:23, vgg16_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 1633.949898, 7.0030739307403564
|
||||
2021-07-23 13:39:30, vgg16_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 219.241699, 6.388982057571411
|
||||
2021-07-23 13:39:37, vgg16_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 286.524128, 7.019291400909424
|
||||
2021-07-23 13:39:47, vgg16_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 189.483289, 10.49472975730896
|
||||
2021-07-23 13:39:56, vgg19_0.pb, tensorflow, cortexA76cpu_tflite21, 1.0, 2055.622028, 7.141560077667236
|
||||
2021-07-23 13:40:03, vgg19_0.pb, tensorflow, adreno640gpu_tflite21, 1.0, 279.573790, 6.552017450332642
|
||||
2021-07-23 13:40:10, vgg19_0.pb, tensorflow, adreno630gpu_tflite21, 1.0, 366.644441, 7.102701663970947
|
||||
2021-07-23 13:40:22, vgg19_0.pb, tensorflow, myriadvpu_openvino2019r2, 1.0, 238.212347, 11.853462934494019
|
||||
2021-07-23 13:40:30, alexnet_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 96.571261, 6.628089427947998
|
||||
2021-07-23 13:40:35, alexnet_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 13.126684, 5.028747081756592
|
||||
2021-07-23 13:40:41, alexnet_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 16.867286, 5.751234292984009
|
||||
2021-07-23 13:40:50, alexnet_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 20.813834, 8.863120794296265
|
||||
2021-07-23 13:40:58, densenet_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 917.205941, 6.2440505027771
|
||||
2021-07-23 13:41:03, densenet_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 186.560380, 5.426648855209351
|
||||
2021-07-23 13:41:09, densenet_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 193.079583, 5.85806941986084
|
||||
2021-07-23 13:41:19, densenet_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 144.107864, 9.45836591720581
|
||||
2021-07-23 13:41:26, googlenet_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 167.581581, 5.3628990650177
|
||||
2021-07-23 13:41:30, googlenet_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 32.758087, 4.164647817611694
|
||||
2021-07-23 13:41:35, googlenet_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 37.961248, 4.700413942337036
|
||||
2021-07-23 13:41:44, googlenet_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 17.089749, 8.949456214904785
|
||||
2021-07-23 13:41:51, mnasnet_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 45.805557, 5.3580002784729
|
||||
2021-07-23 13:41:55, mnasnet_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 11.630591, 4.193651914596558
|
||||
2021-07-23 13:42:00, mnasnet_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 11.726332, 4.819653034210205
|
||||
2021-07-23 13:42:08, mnasnet_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 24.352560, 8.22120213508606
|
||||
2021-07-23 13:42:15, mobilenetv2_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 43.963505, 5.480044364929199
|
||||
2021-07-23 13:42:19, mobilenetv2_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 9.920667, 3.9429852962493896
|
||||
2021-07-23 13:42:24, mobilenetv2_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 9.986092, 4.7738893032073975
|
||||
2021-07-23 13:42:32, mobilenetv2_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 22.018673, 7.977810621261597
|
||||
2021-07-23 13:42:39, mobilenetv3large_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 55.177037, 5.249249696731567
|
||||
2021-07-23 13:42:44, mobilenetv3large_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 12.548915, 4.7999231815338135
|
||||
2021-07-23 13:42:49, mobilenetv3large_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 12.926972, 4.916620969772339
|
||||
2021-07-23 13:42:58, mobilenetv3large_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 56.030651, 8.95370864868164
|
||||
2021-07-23 13:43:05, mobilenetv3small_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 29.850499, 5.442401170730591
|
||||
2021-07-23 13:43:09, mobilenetv3small_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 6.705541, 3.8803601264953613
|
||||
2021-07-23 13:43:14, mobilenetv3small_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 7.005715, 5.112581968307495
|
||||
2021-07-23 13:43:22, mobilenetv3small_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 39.163780, 7.835463523864746
|
||||
2021-07-23 13:43:29, resnet18_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 216.197146, 5.299234867095947
|
||||
2021-07-23 13:43:33, resnet18_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 39.323517, 3.87131404876709
|
||||
2021-07-23 13:43:37, resnet18_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 49.428744, 4.355535984039307
|
||||
2021-07-23 13:43:45, resnet18_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 21.886077, 7.955303907394409
|
||||
2021-07-23 13:43:52, shufflenetv2_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 21.318069, 5.522752285003662
|
||||
2021-07-23 13:43:56, shufflenetv2_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 5.423899, 3.8563084602355957
|
||||
2021-07-23 13:44:01, shufflenetv2_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 5.579220, 4.517962455749512
|
||||
2021-07-23 13:44:09, shufflenetv2_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 20.828206, 8.633913040161133
|
||||
2021-07-23 13:44:16, squeezenet_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 98.376599, 5.07289457321167
|
||||
2021-07-23 13:44:20, squeezenet_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 18.674224, 3.8373663425445557
|
||||
2021-07-23 13:44:24, squeezenet_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 21.094962, 4.381319522857666
|
||||
2021-07-23 13:44:32, squeezenet_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 11.052252, 7.76660680770874
|
||||
2021-07-23 13:44:41, vgg16_0.onnx, onnx, cortexA76cpu_tflite21, 1.0, 1668.508478, 7.156689405441284
|
||||
2021-07-23 13:44:47, vgg16_0.onnx, onnx, adreno640gpu_tflite21, 1.0, 219.264772, 5.871518135070801
|
||||
2021-07-23 13:44:54, vgg16_0.onnx, onnx, adreno630gpu_tflite21, 1.0, 286.599755, 6.5662453174591064
|
||||
2021-07-23 13:45:04, vgg16_0.onnx, onnx, myriadvpu_openvino2019r2, 1.0, 211.373515, 10.228150129318237
|
||||
Загрузка…
Ссылка в новой задаче