Benchmarks: Code Revision - change 'reduce' to 'reduce_op' (#156)
**Description** Change the field name `reduce` to `reduce_op`.
This commit is contained in:
Родитель
783c91258d
Коммит
7293e783f1
|
@ -44,7 +44,7 @@ This chapter will describe the design details of all the components in `benchmar
|
|||
|
||||
The E2E model benchmarks have 4-layer inheritance relationship.
|
||||
|
||||
#### Training
|
||||
#### Training
|
||||
|
||||
The general process of model training is:
|
||||
|
||||
|
@ -66,7 +66,7 @@ Compared with training, it just gets rid of create_optimizer operation.
|
|||
|
||||
### Micro Benchmarks
|
||||
|
||||
The micro-benchmarks have 3-layer Inheritance Relationship. There are two base classes for micro-benchmark:
|
||||
The micro-benchmarks have 3-layer Inheritance Relationship. There are two base classes for micro-benchmark:
|
||||
`MicroBenchmark` is pure-python benchmark.
|
||||
`MicroBenchmarkWithInvoke` is benchmark depending on third-party executable program.
|
||||
|
||||
|
@ -219,7 +219,7 @@ result = {
|
|||
...
|
||||
'metricsM': List[Number],
|
||||
},
|
||||
'reduce': {
|
||||
'reduce_op': {
|
||||
'metrics1': ReduceType,
|
||||
...
|
||||
'metricsM': ReduceType,
|
||||
|
@ -249,7 +249,7 @@ result = {
|
|||
'throughput-inference-float32': [avg_throughput1, ..., avg_throughputN],
|
||||
'throughput-inference-float16': [avg_throughput1, ..., avg_throughputN],
|
||||
},
|
||||
'reduce': {
|
||||
'reduce_op': {
|
||||
'throughput-train-float32': 'min',
|
||||
'throughput-train-float16': 'min',
|
||||
'throughput-inference-float32': None,
|
||||
|
@ -272,7 +272,7 @@ result = {
|
|||
'result': { # Key is metrics
|
||||
'overhead': [overhead1, ..., overheadN],
|
||||
},
|
||||
'reduce': {
|
||||
'reduce_op': {
|
||||
'overhead': None,
|
||||
},
|
||||
}
|
||||
|
|
|
@ -31,7 +31,7 @@ class BenchmarkResult():
|
|||
self.__end_time = None
|
||||
self.__raw_data = dict()
|
||||
self.__result = dict()
|
||||
self.__reduce = dict()
|
||||
self.__reduce_op = dict()
|
||||
|
||||
def __eq__(self, rhs):
|
||||
"""Override equal function for deep comparison.
|
||||
|
@ -89,7 +89,7 @@ class BenchmarkResult():
|
|||
|
||||
if metric not in self.__result:
|
||||
self.__result[metric] = list()
|
||||
self.__reduce[metric] = reduce_type.value if isinstance(reduce_type, Enum) else None
|
||||
self.__reduce_op[metric] = reduce_type.value if isinstance(reduce_type, Enum) else None
|
||||
self.__result[metric].append(value)
|
||||
|
||||
return True
|
||||
|
@ -177,3 +177,8 @@ class BenchmarkResult():
|
|||
def result(self):
|
||||
"""Decoration function to access __result."""
|
||||
return self.__result
|
||||
|
||||
@property
|
||||
def reduce_op(self):
|
||||
"""Decoration function to access __reduce_op."""
|
||||
return self.__reduce_op
|
||||
|
|
|
@ -220,7 +220,7 @@ def test_train():
|
|||
'"steptime_train_float32": [[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]], '
|
||||
'"throughput_train_float32": [[16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0]]}, '
|
||||
'"result": {"steptime_train_float32": [2.0], "throughput_train_float32": [16000.0]}, '
|
||||
'"reduce": {"steptime_train_float32": "max", "throughput_train_float32": "min"}}'
|
||||
'"reduce_op": {"steptime_train_float32": "max", "throughput_train_float32": "min"}}'
|
||||
)
|
||||
assert (benchmark._preprocess())
|
||||
assert (benchmark._ModelBenchmark__train(Precision.FLOAT32))
|
||||
|
@ -230,7 +230,7 @@ def test_train():
|
|||
benchmark = create_benchmark('--num_steps 0')
|
||||
expected_result = (
|
||||
'{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 3, '
|
||||
'"start_time": null, "end_time": null, "raw_data": {}, "result": {}, "reduce": {}}'
|
||||
'"start_time": null, "end_time": null, "raw_data": {}, "result": {}, "reduce_op": {}}'
|
||||
)
|
||||
assert (benchmark._preprocess())
|
||||
assert (benchmark._ModelBenchmark__train(Precision.FLOAT32) is False)
|
||||
|
@ -246,7 +246,7 @@ def test_inference():
|
|||
'"steptime_inference_float16": [[4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0]], '
|
||||
'"throughput_inference_float16": [[8000.0, 8000.0, 8000.0, 8000.0, 8000.0, 8000.0, 8000.0, 8000.0]]}, '
|
||||
'"result": {"steptime_inference_float16": [4.0], "throughput_inference_float16": [8000.0]}, '
|
||||
'"reduce": {"steptime_inference_float16": null, "throughput_inference_float16": null}}'
|
||||
'"reduce_op": {"steptime_inference_float16": null, "throughput_inference_float16": null}}'
|
||||
)
|
||||
assert (benchmark._preprocess())
|
||||
assert (benchmark._ModelBenchmark__inference(Precision.FLOAT16))
|
||||
|
@ -256,7 +256,7 @@ def test_inference():
|
|||
benchmark = create_benchmark('--num_steps 0')
|
||||
expected_result = (
|
||||
'{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 3, '
|
||||
'"start_time": null, "end_time": null, "raw_data": {}, "result": {}, "reduce": {}}'
|
||||
'"start_time": null, "end_time": null, "raw_data": {}, "result": {}, "reduce_op": {}}'
|
||||
)
|
||||
assert (benchmark._preprocess())
|
||||
assert (benchmark._ModelBenchmark__inference(Precision.FLOAT16) is False)
|
||||
|
@ -296,7 +296,7 @@ def test_benchmark():
|
|||
'"throughput_train_float16": [[16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0]]}, '
|
||||
'"result": {"steptime_train_float32": [2.0], "throughput_train_float32": [16000.0], '
|
||||
'"steptime_train_float16": [2.0], "throughput_train_float16": [16000.0]}, '
|
||||
'"reduce": {"steptime_train_float32": "max", "throughput_train_float32": "min", '
|
||||
'"reduce_op": {"steptime_train_float32": "max", "throughput_train_float32": "min", '
|
||||
'"steptime_train_float16": "max", "throughput_train_float16": "min"}}'
|
||||
)
|
||||
assert (benchmark.serialized_result == expected_serialized_result)
|
||||
|
|
|
@ -149,7 +149,7 @@ def test_launch_benchmark():
|
|||
'"return_code": 0, "start_time": null, "end_time": null, '
|
||||
'"raw_data": {"accumulation_result": ["1,3,6,10"]}, '
|
||||
'"result": {"accumulation_result": [10]}, '
|
||||
'"reduce": {"accumulation_result": null}}'
|
||||
'"reduce_op": {"accumulation_result": null}}'
|
||||
)
|
||||
assert (result == expected)
|
||||
|
||||
|
@ -173,7 +173,7 @@ def test_launch_benchmark():
|
|||
'"return_code": 0, "start_time": null, "end_time": null, '
|
||||
'"raw_data": {"accumulation_result": ["1,3,6"]}, '
|
||||
'"result": {"accumulation_result": [6]}, '
|
||||
'"reduce": {"accumulation_result": null}}'
|
||||
'"reduce_op": {"accumulation_result": null}}'
|
||||
)
|
||||
assert (result == expected)
|
||||
|
||||
|
|
|
@ -83,6 +83,6 @@ def test_serialize_deserialize():
|
|||
'"start_time": "2021-02-03 16:59:49", "end_time": "2021-02-03 17:00:08", '
|
||||
'"raw_data": {"metric1": [[1, 2, 3], [4, 5, 6], [7, 8, 9]]}, '
|
||||
'"result": {"metric1": [300, 200], "metric2": [100]}, '
|
||||
'"reduce": {"metric1": "max", "metric2": "avg"}}'
|
||||
'"reduce_op": {"metric1": "max", "metric2": "avg"}}'
|
||||
)
|
||||
assert (result.to_string() == expected)
|
||||
|
|
Загрузка…
Ссылка в новой задаче