172 строки
7.3 KiB
Python
172 строки
7.3 KiB
Python
# Copyright (c) Microsoft Corporation.
|
|
# Licensed under the MIT License.
|
|
|
|
"""Tests for ResultSummary module."""
|
|
|
|
import unittest
|
|
import yaml
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
from superbench.analyzer import ResultSummary
|
|
import superbench.analyzer.file_handler as file_handler
|
|
|
|
|
|
class TestResultSummary(unittest.TestCase):
|
|
"""Test for ResultSummary class."""
|
|
def setUp(self):
|
|
"""Method called to prepare the test fixture."""
|
|
self.parent_path = Path(__file__).parent
|
|
self.output_excel_file = str(self.parent_path / 'results-summary.xlsx')
|
|
self.output_md_file = str(self.parent_path / 'results-summary.md')
|
|
self.output_html_file = str(self.parent_path / 'results-summary.html')
|
|
self.test_rule_file_fake = str(self.parent_path / 'test_rules_fake.yaml')
|
|
self.test_raw_data = str(self.parent_path / 'test_results.jsonl')
|
|
self.test_rule_file = str(self.parent_path / 'test_summary_rules.yaml')
|
|
|
|
def tearDown(self):
|
|
"""Method called after the test method has been called and the result recorded."""
|
|
for file in [self.output_excel_file, self.test_rule_file_fake, self.output_md_file, self.output_html_file]:
|
|
p = Path(file)
|
|
if p.is_file():
|
|
p.unlink()
|
|
|
|
def test_result_summary(self):
|
|
"""Test result summary class."""
|
|
rs1 = ResultSummary()
|
|
rs1._raw_data_df = file_handler.read_raw_data(self.test_raw_data)
|
|
rs1._benchmark_metrics_dict = rs1._get_metrics_by_benchmarks(list(rs1._raw_data_df))
|
|
# Test - _check_rules
|
|
# Negative case
|
|
false_rules = [
|
|
{
|
|
'categories': 'KernelLaunch',
|
|
'metrics': ['kernel-launch/event_overhead:\\d+']
|
|
}, {
|
|
'categories': 'KernelLaunch',
|
|
'statistics': 'abb',
|
|
'metrics': ['kernel-launch/event_overhead:\\d+']
|
|
}, {
|
|
'categories': 'KernelLaunch',
|
|
'statistics': 'mean',
|
|
'metrics': ['kernel-launch/event_overhead:\\d+'],
|
|
'aggregate': 'abb'
|
|
}
|
|
]
|
|
metric = 'kernel-launch/event_overhead:0'
|
|
for rules in false_rules:
|
|
self.assertRaises(Exception, rs1._check_rules, rules, metric)
|
|
# Positive case
|
|
true_rules = [
|
|
{
|
|
'categories': 'KernelLaunch',
|
|
'statistics': 'mean',
|
|
'metrics': ['kernel-launch/event_overhead:\\d+'],
|
|
'aggregate': True
|
|
},
|
|
{
|
|
'categories': 'KernelLaunch',
|
|
'statistics': ['mean', 'p50'],
|
|
'metrics': ['kernel-launch/event_overhead:\\d+']
|
|
},
|
|
{
|
|
'categories': 'KernelLaunch',
|
|
'statistics': 'mean',
|
|
'metrics': ['kernel-launch/event_overhead:\\d+'],
|
|
'aggregate': 'kernel-launch/event_overhead(:\\d+)'
|
|
},
|
|
]
|
|
for rules in true_rules:
|
|
assert (rs1._check_rules(rules, metric))
|
|
|
|
# Test - _parse_rules
|
|
# Negative case
|
|
rs2 = ResultSummary()
|
|
self.assertRaises(Exception, file_handler.read_rules, self.test_rule_file_fake)
|
|
rs2._raw_data_df = file_handler.read_raw_data(self.test_raw_data)
|
|
rs2._benchmark_metrics_dict = rs2._get_metrics_by_benchmarks(list(rs2._raw_data_df))
|
|
p = Path(self.test_rule_file)
|
|
with p.open() as f:
|
|
rules = yaml.load(f, Loader=yaml.SafeLoader)
|
|
rules['superbench']['rules']['fake'] = false_rules[0]
|
|
with open(self.test_rule_file_fake, 'w') as f:
|
|
yaml.dump(rules, f)
|
|
assert (rs1._parse_rules([]) is False)
|
|
# Positive case
|
|
rules = file_handler.read_rules(self.test_rule_file)
|
|
assert (rs1._parse_rules(rules))
|
|
|
|
# Test - _generate_summary
|
|
summary = rs1._generate_summary(round=2)
|
|
assert (len(summary) == 3)
|
|
|
|
# Test - _merge_summary
|
|
expected_summary_merge = [
|
|
['KernelLaunch', 'kernel-launch/event_overhead', 'mean', 0.0097],
|
|
['KernelLaunch', 'kernel-launch/event_overhead', 'p90', 0.006],
|
|
['KernelLaunch', 'kernel-launch/event_overhead', 'min', 0.0055],
|
|
['KernelLaunch', 'kernel-launch/event_overhead', 'max', 0.1],
|
|
['KernelLaunch', 'kernel-launch/wall_overhead', 'mean', 0.01],
|
|
['KernelLaunch', 'kernel-launch/wall_overhead', 'p90', 0.011],
|
|
['KernelLaunch', 'kernel-launch/wall_overhead', 'min', 0.01],
|
|
['KernelLaunch', 'kernel-launch/wall_overhead', 'max', 0.011],
|
|
['NCCL', 'nccl-bw/allreduce_8388608_busbw:0', 'mean', 89.51],
|
|
['RDMA', 'ib-loopback/IB_write_8388608_Avg_*:0', 'mean', 23925.84]
|
|
]
|
|
expected_summary_merge_df = pd.DataFrame(expected_summary_merge)
|
|
summary_merge_df = rs1._merge_summary(summary)
|
|
pd.testing.assert_frame_equal(expected_summary_merge_df, summary_merge_df)
|
|
|
|
def test_no_matched_rule(self):
|
|
"""Test for support no matching rules."""
|
|
# Positive case
|
|
rules = {
|
|
'superbench': {
|
|
'rules': {
|
|
'fake': {
|
|
'categories': 'FAKE',
|
|
'statistics': ['mean', 'max'],
|
|
'metrics': ['abb/fake:\\d+'],
|
|
'aggregate': True
|
|
}
|
|
}
|
|
}
|
|
}
|
|
rs1 = ResultSummary()
|
|
rs1._raw_data_df = file_handler.read_raw_data(self.test_raw_data)
|
|
rs1._benchmark_metrics_dict = rs1._get_metrics_by_benchmarks(list(rs1._raw_data_df))
|
|
assert (rs1._parse_rules(rules))
|
|
summary = rs1._generate_summary(round=2)
|
|
assert (len(summary) == 1)
|
|
assert (summary['FAKE'] == [['FAKE', '', 'mean', ''], ['FAKE', '', 'max', '']])
|
|
|
|
def test_result_summary_run(self):
|
|
"""Test for the run process of result summary."""
|
|
# Test - output in excel
|
|
ResultSummary().run(self.test_raw_data, self.test_rule_file, str(self.parent_path), 'excel', round=2)
|
|
excel_file = pd.ExcelFile(self.output_excel_file, engine='openpyxl')
|
|
data_sheet_name = 'Summary'
|
|
summary = excel_file.parse(data_sheet_name, header=None)
|
|
expect_result_file = pd.ExcelFile(str(self.parent_path / '../data/results_summary.xlsx'), engine='openpyxl')
|
|
expect_result = expect_result_file.parse(data_sheet_name, header=None)
|
|
pd.testing.assert_frame_equal(summary, expect_result)
|
|
|
|
# Test - output in md
|
|
ResultSummary().run(self.test_raw_data, self.test_rule_file, str(self.parent_path), 'md', round=2)
|
|
expected_md_file = str(self.parent_path / '../data/results_summary.md')
|
|
with open(expected_md_file, 'r') as f:
|
|
expect_result = f.read()
|
|
with open(self.output_md_file, 'r') as f:
|
|
summary = f.read()
|
|
assert (summary == expect_result)
|
|
|
|
# Test - output in html
|
|
ResultSummary().run(self.test_raw_data, self.test_rule_file, str(self.parent_path), 'html', round=2)
|
|
expected_html_file = str(self.parent_path / '../data/results_summary.html')
|
|
with open(expected_html_file, 'r') as f:
|
|
expect_result = f.read()
|
|
with open(self.output_html_file, 'r') as f:
|
|
summary = f.read()
|
|
assert (summary == expect_result)
|