1409 строки
72 KiB
JSON
1409 строки
72 KiB
JSON
[
|
|
{
|
|
"index": "sb-validation-01",
|
|
"diagnosis/accept": false,
|
|
"diagnosis/issue_num": 1,
|
|
"diagnosis/category": "KernelLaunch",
|
|
"diagnosis/issue_details": "kernel-launch/event_overhead:0(B/L: 0.0060 VAL: 0.1000 VAR: 1577.85% Rule:lambda x:x>0.05)",
|
|
"bert_models/pytorch-bert-base/steptime_train_float32": 114.5916701062,
|
|
"bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591,
|
|
"bert_models/pytorch-bert-base/steptime_train_float16": 83.8895108318,
|
|
"bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582742,
|
|
"bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371914,
|
|
"bert_models/pytorch-bert-large/throughput_train_float32": 103.9487609742,
|
|
"bert_models/pytorch-bert-large/steptime_train_float16": 206.8114168942,
|
|
"bert_models/pytorch-bert-large/throughput_train_float16": 154.8408911711,
|
|
"pytorch-computation-communication-overlap/mul_cost:0": 44.1822062144,
|
|
"pytorch-computation-communication-overlap/mul_cost:1": 44.1822139389,
|
|
"pytorch-computation-communication-overlap/mul_cost:2": 43.9701470781,
|
|
"pytorch-computation-communication-overlap/mul_cost:3": 43.9701478756,
|
|
"pytorch-computation-communication-overlap/mul_cost:4": 43.9701779317,
|
|
"pytorch-computation-communication-overlap/mul_cost:5": 43.9701571606,
|
|
"pytorch-computation-communication-overlap/mul_cost:6": 43.9701651983,
|
|
"pytorch-computation-communication-overlap/mul_cost:7": 44.1795444785,
|
|
"pytorch-computation-communication-overlap/matmul_cost:0": 137.0477370556,
|
|
"pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344693,
|
|
"pytorch-computation-communication-overlap/matmul_cost:2": 137.047772209,
|
|
"pytorch-computation-communication-overlap/matmul_cost:3": 137.0477969726,
|
|
"pytorch-computation-communication-overlap/matmul_cost:4": 137.0481367431,
|
|
"pytorch-computation-communication-overlap/matmul_cost:5": 137.0482198877,
|
|
"pytorch-computation-communication-overlap/matmul_cost:6": 137.0477532237,
|
|
"pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081607,
|
|
"densenet_models/pytorch-densenet169/steptime_train_float32": 150.6415554322,
|
|
"densenet_models/pytorch-densenet169/throughput_train_float32": 212.4701719243,
|
|
"densenet_models/pytorch-densenet169/steptime_train_float16": 145.3081957297,
|
|
"densenet_models/pytorch-densenet169/throughput_train_float16": 220.2932337305,
|
|
"densenet_models/pytorch-densenet201/steptime_train_float32": 182.9111778643,
|
|
"densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232,
|
|
"densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596,
|
|
"densenet_models/pytorch-densenet201/throughput_train_float16": 181.5446573603,
|
|
"gemm-flops/FP64:0": 9031.23,
|
|
"gemm-flops/FP64:1": 9040.85,
|
|
"gemm-flops/FP64:2": 9010.56,
|
|
"gemm-flops/FP64:3": 9041.26,
|
|
"gemm-flops/FP64:4": 9039.19,
|
|
"gemm-flops/FP64:5": 9015.69,
|
|
"gemm-flops/FP64:6": 9022.19,
|
|
"gemm-flops/FP64:7": 9030.2,
|
|
"gemm-flops/FP32:0": 18362.1,
|
|
"gemm-flops/FP32:1": 18375.6,
|
|
"gemm-flops/FP32:2": 18314.9,
|
|
"gemm-flops/FP32:3": 18375.6,
|
|
"gemm-flops/FP32:4": 18368.6,
|
|
"gemm-flops/FP32:5": 18347.1,
|
|
"gemm-flops/FP32:6": 18247.4,
|
|
"gemm-flops/FP32:7": 18318.4,
|
|
"gemm-flops/FP16:0": 33878,
|
|
"gemm-flops/FP16:1": 33911.1,
|
|
"gemm-flops/FP16:2": 33769.3,
|
|
"gemm-flops/FP16:3": 33909.9,
|
|
"gemm-flops/FP16:4": 33896.5,
|
|
"gemm-flops/FP16:5": 33798.1,
|
|
"gemm-flops/FP16:6": 33647.3,
|
|
"gemm-flops/FP16:7": 33764.8,
|
|
"gemm-flops/FP64_TC:0": 18963.6,
|
|
"gemm-flops/FP64_TC:1": 18924.2,
|
|
"gemm-flops/FP64_TC:2": 18930.3,
|
|
"gemm-flops/FP64_TC:3": 18971.9,
|
|
"gemm-flops/FP64_TC:4": 18946,
|
|
"gemm-flops/FP64_TC:5": 18945,
|
|
"gemm-flops/FP64_TC:6": 18822.9,
|
|
"gemm-flops/FP64_TC:7": 18911.1,
|
|
"gemm-flops/TF32_TC:0": 127900,
|
|
"gemm-flops/TF32_TC:1": 129094,
|
|
"gemm-flops/TF32_TC:2": 127831,
|
|
"gemm-flops/TF32_TC:3": 128709,
|
|
"gemm-flops/TF32_TC:4": 127388,
|
|
"gemm-flops/TF32_TC:5": 127861,
|
|
"gemm-flops/TF32_TC:6": 128492,
|
|
"gemm-flops/TF32_TC:7": 127720,
|
|
"gemm-flops/BF16_TC:0": 264965,
|
|
"gemm-flops/BF16_TC:1": 266638,
|
|
"gemm-flops/BF16_TC:2": 263151,
|
|
"gemm-flops/BF16_TC:3": 264752,
|
|
"gemm-flops/BF16_TC:4": 263049,
|
|
"gemm-flops/BF16_TC:5": 266605,
|
|
"gemm-flops/BF16_TC:6": 267501,
|
|
"gemm-flops/BF16_TC:7": 263880,
|
|
"gemm-flops/FP16_TC:0": 279474,
|
|
"gemm-flops/FP16_TC:1": 281256,
|
|
"gemm-flops/FP16_TC:2": 277403,
|
|
"gemm-flops/FP16_TC:3": 279147,
|
|
"gemm-flops/FP16_TC:4": 277587,
|
|
"gemm-flops/FP16_TC:5": 281537,
|
|
"gemm-flops/FP16_TC:6": 282132,
|
|
"gemm-flops/FP16_TC:7": 277788,
|
|
"gemm-flops/INT8_TC:0": 475160,
|
|
"gemm-flops/INT8_TC:1": 477725,
|
|
"gemm-flops/INT8_TC:2": 471621,
|
|
"gemm-flops/INT8_TC:3": 473716,
|
|
"gemm-flops/INT8_TC:4": 472124,
|
|
"gemm-flops/INT8_TC:5": 479972,
|
|
"gemm-flops/INT8_TC:6": 481327,
|
|
"gemm-flops/INT8_TC:7": 474710,
|
|
"gemm-flops/INT4_TC:0": 970330,
|
|
"gemm-flops/INT4_TC:1": 976837,
|
|
"gemm-flops/INT4_TC:2": 966003,
|
|
"gemm-flops/INT4_TC:3": 971315,
|
|
"gemm-flops/INT4_TC:4": 964441,
|
|
"gemm-flops/INT4_TC:5": 982461,
|
|
"gemm-flops/INT4_TC:6": 979610,
|
|
"gemm-flops/INT4_TC:7": 968359,
|
|
"gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971836,
|
|
"gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.1154543969,
|
|
"gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742235,
|
|
"gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.1394499411,
|
|
"gpu-sm-copy-bw/dtoh:0": 3.91755,
|
|
"gpu-sm-copy-bw/dtoh:1": 4.45414,
|
|
"gpu-sm-copy-bw/dtoh:2": 1.26483,
|
|
"gpu-sm-copy-bw/dtoh:3": 1.30041,
|
|
"gpu-sm-copy-bw/dtoh:4": 1.31577,
|
|
"gpu-sm-copy-bw/dtoh:5": 1.27968,
|
|
"gpu-sm-copy-bw/dtoh:6": 4.47849,
|
|
"gpu-sm-copy-bw/dtoh:7": 3.96231,
|
|
"gpu-sm-copy-bw/dtoh:8": 3.91705,
|
|
"gpu-sm-copy-bw/dtoh:9": 4.45487,
|
|
"gpu-sm-copy-bw/dtoh:10": 1.26352,
|
|
"gpu-sm-copy-bw/dtoh:11": 1.2999,
|
|
"gpu-sm-copy-bw/dtoh:12": 1.31677,
|
|
"gpu-sm-copy-bw/dtoh:13": 1.27885,
|
|
"gpu-sm-copy-bw/dtoh:14": 4.47913,
|
|
"gpu-sm-copy-bw/dtoh:15": 3.95893,
|
|
"gpu-sm-copy-bw/dtoh:16": 3.91729,
|
|
"gpu-sm-copy-bw/dtoh:17": 4.45627,
|
|
"gpu-sm-copy-bw/dtoh:18": 1.26437,
|
|
"gpu-sm-copy-bw/dtoh:19": 1.30144,
|
|
"gpu-sm-copy-bw/dtoh:20": 1.31704,
|
|
"gpu-sm-copy-bw/dtoh:21": 1.27857,
|
|
"gpu-sm-copy-bw/dtoh:22": 4.47889,
|
|
"gpu-sm-copy-bw/dtoh:23": 3.95984,
|
|
"gpu-sm-copy-bw/dtoh:24": 3.92025,
|
|
"gpu-sm-copy-bw/dtoh:25": 4.45423,
|
|
"gpu-sm-copy-bw/dtoh:26": 1.26449,
|
|
"gpu-sm-copy-bw/dtoh:27": 1.29954,
|
|
"gpu-sm-copy-bw/dtoh:28": 1.31731,
|
|
"gpu-sm-copy-bw/dtoh:29": 1.27916,
|
|
"gpu-sm-copy-bw/dtoh:30": 4.4797,
|
|
"gpu-sm-copy-bw/dtoh:31": 3.96124,
|
|
"gpu-sm-copy-bw/htod:0": 23.9685,
|
|
"gpu-sm-copy-bw/htod:1": 23.967,
|
|
"gpu-sm-copy-bw/htod:2": 19.9898,
|
|
"gpu-sm-copy-bw/htod:3": 20.2848,
|
|
"gpu-sm-copy-bw/htod:4": 20.3931,
|
|
"gpu-sm-copy-bw/htod:5": 20.0888,
|
|
"gpu-sm-copy-bw/htod:6": 23.9766,
|
|
"gpu-sm-copy-bw/htod:7": 23.9792,
|
|
"gpu-sm-copy-bw/htod:8": 23.9497,
|
|
"gpu-sm-copy-bw/htod:9": 23.9438,
|
|
"gpu-sm-copy-bw/htod:10": 20.0039,
|
|
"gpu-sm-copy-bw/htod:11": 20.2469,
|
|
"gpu-sm-copy-bw/htod:12": 20.4344,
|
|
"gpu-sm-copy-bw/htod:13": 20.1005,
|
|
"gpu-sm-copy-bw/htod:14": 23.9491,
|
|
"gpu-sm-copy-bw/htod:15": 23.9898,
|
|
"gpu-sm-copy-bw/htod:16": 23.962,
|
|
"gpu-sm-copy-bw/htod:17": 23.97,
|
|
"gpu-sm-copy-bw/htod:18": 19.987,
|
|
"gpu-sm-copy-bw/htod:19": 20.1549,
|
|
"gpu-sm-copy-bw/htod:20": 20.3931,
|
|
"gpu-sm-copy-bw/htod:21": 20.1013,
|
|
"gpu-sm-copy-bw/htod:22": 23.9865,
|
|
"gpu-sm-copy-bw/htod:23": 23.972,
|
|
"gpu-sm-copy-bw/htod:24": 23.9569,
|
|
"gpu-sm-copy-bw/htod:25": 23.9762,
|
|
"gpu-sm-copy-bw/htod:26": 19.92,
|
|
"gpu-sm-copy-bw/htod:27": 20.2638,
|
|
"gpu-sm-copy-bw/htod:28": 20.4419,
|
|
"gpu-sm-copy-bw/htod:29": 20.1054,
|
|
"gpu-sm-copy-bw/htod:30": 23.9752,
|
|
"gpu-sm-copy-bw/htod:31": 23.9631,
|
|
"ib-loopback/IB_write_512_Avg_0:0": 1492.89,
|
|
"ib-loopback/IB_write_1024_Avg_0:0": 3224.92,
|
|
"ib-loopback/IB_write_2048_Avg_0:0": 6714.5,
|
|
"ib-loopback/IB_write_4096_Avg_0:0": 12871.93,
|
|
"ib-loopback/IB_write_8192_Avg_0:0": 19990.78,
|
|
"ib-loopback/IB_write_16384_Avg_0:0": 22172.25,
|
|
"ib-loopback/IB_write_32768_Avg_0:0": 23073.19,
|
|
"ib-loopback/IB_write_65536_Avg_0:0": 23527.09,
|
|
"ib-loopback/IB_write_131072_Avg_0:0": 23805.92,
|
|
"ib-loopback/IB_write_262144_Avg_0:0": 23380.51,
|
|
"ib-loopback/IB_write_524288_Avg_0:0": 23856.63,
|
|
"ib-loopback/IB_write_1048576_Avg_0:0": 23869.58,
|
|
"ib-loopback/IB_write_2097152_Avg_0:0": 23885.7,
|
|
"ib-loopback/IB_write_4194304_Avg_0:0": 23914.56,
|
|
"ib-loopback/IB_write_8388608_Avg_0:0": 23935.21,
|
|
"ib-loopback/IB_write_512_Avg_1:0": 1479.1,
|
|
"ib-loopback/IB_write_1024_Avg_1:0": 3246.48,
|
|
"ib-loopback/IB_write_2048_Avg_1:0": 6754.94,
|
|
"ib-loopback/IB_write_4096_Avg_1:0": 13101.09,
|
|
"ib-loopback/IB_write_8192_Avg_1:0": 19945.81,
|
|
"ib-loopback/IB_write_16384_Avg_1:0": 22631.62,
|
|
"ib-loopback/IB_write_32768_Avg_1:0": 23381.58,
|
|
"ib-loopback/IB_write_65536_Avg_1:0": 23515.29,
|
|
"ib-loopback/IB_write_131072_Avg_1:0": 23756.01,
|
|
"ib-loopback/IB_write_262144_Avg_1:0": 23875.91,
|
|
"ib-loopback/IB_write_524288_Avg_1:0": 23911.13,
|
|
"ib-loopback/IB_write_1048576_Avg_1:0": 23935.42,
|
|
"ib-loopback/IB_write_2097152_Avg_1:0": 23941.25,
|
|
"ib-loopback/IB_write_4194304_Avg_1:0": 23922.06,
|
|
"ib-loopback/IB_write_8388608_Avg_1:0": 23928.59,
|
|
"ib-loopback/IB_write_512_Avg_2:0": 1505.29,
|
|
"ib-loopback/IB_write_1024_Avg_2:0": 3215.97,
|
|
"ib-loopback/IB_write_2048_Avg_2:0": 6745.49,
|
|
"ib-loopback/IB_write_4096_Avg_2:0": 11548.88,
|
|
"ib-loopback/IB_write_8192_Avg_2:0": 19432.15,
|
|
"ib-loopback/IB_write_16384_Avg_2:0": 22765.51,
|
|
"ib-loopback/IB_write_32768_Avg_2:0": 23235.07,
|
|
"ib-loopback/IB_write_65536_Avg_2:0": 23620.08,
|
|
"ib-loopback/IB_write_131072_Avg_2:0": 23759.08,
|
|
"ib-loopback/IB_write_262144_Avg_2:0": 23859.82,
|
|
"ib-loopback/IB_write_524288_Avg_2:0": 23775.01,
|
|
"ib-loopback/IB_write_1048576_Avg_2:0": 23885.96,
|
|
"ib-loopback/IB_write_2097152_Avg_2:0": 23894.73,
|
|
"ib-loopback/IB_write_4194304_Avg_2:0": 23909.98,
|
|
"ib-loopback/IB_write_8388608_Avg_2:0": 23927.21,
|
|
"ib-loopback/IB_write_512_Avg_3:0": 1474.47,
|
|
"ib-loopback/IB_write_1024_Avg_3:0": 3344.68,
|
|
"ib-loopback/IB_write_2048_Avg_3:0": 6606.35,
|
|
"ib-loopback/IB_write_4096_Avg_3:0": 12071.4,
|
|
"ib-loopback/IB_write_8192_Avg_3:0": 18408.33,
|
|
"ib-loopback/IB_write_16384_Avg_3:0": 20789.27,
|
|
"ib-loopback/IB_write_32768_Avg_3:0": 22469.43,
|
|
"ib-loopback/IB_write_65536_Avg_3:0": 22777.1,
|
|
"ib-loopback/IB_write_131072_Avg_3:0": 23461.03,
|
|
"ib-loopback/IB_write_262144_Avg_3:0": 23397.19,
|
|
"ib-loopback/IB_write_524288_Avg_3:0": 23526.55,
|
|
"ib-loopback/IB_write_1048576_Avg_3:0": 23854.76,
|
|
"ib-loopback/IB_write_2097152_Avg_3:0": 23862.23,
|
|
"ib-loopback/IB_write_4194304_Avg_3:0": 23931.15,
|
|
"ib-loopback/IB_write_8388608_Avg_3:0": 23924.44,
|
|
"ib-loopback/IB_write_512_Avg_4:0": 1523.33,
|
|
"ib-loopback/IB_write_1024_Avg_4:0": 3233.23,
|
|
"ib-loopback/IB_write_2048_Avg_4:0": 6792.88,
|
|
"ib-loopback/IB_write_4096_Avg_4:0": 12616.05,
|
|
"ib-loopback/IB_write_8192_Avg_4:0": 19324.05,
|
|
"ib-loopback/IB_write_16384_Avg_4:0": 22082.51,
|
|
"ib-loopback/IB_write_32768_Avg_4:0": 23294.23,
|
|
"ib-loopback/IB_write_65536_Avg_4:0": 23546.22,
|
|
"ib-loopback/IB_write_131072_Avg_4:0": 23727.91,
|
|
"ib-loopback/IB_write_262144_Avg_4:0": 23843.93,
|
|
"ib-loopback/IB_write_524288_Avg_4:0": 23905.96,
|
|
"ib-loopback/IB_write_1048576_Avg_4:0": 23902.37,
|
|
"ib-loopback/IB_write_2097152_Avg_4:0": 23921.03,
|
|
"ib-loopback/IB_write_4194304_Avg_4:0": 23921.06,
|
|
"ib-loopback/IB_write_8388608_Avg_4:0": 23922.4,
|
|
"ib-loopback/IB_write_512_Avg_5:0": 1506.63,
|
|
"ib-loopback/IB_write_1024_Avg_5:0": 3261.71,
|
|
"ib-loopback/IB_write_2048_Avg_5:0": 6752.38,
|
|
"ib-loopback/IB_write_4096_Avg_5:0": 13592.41,
|
|
"ib-loopback/IB_write_8192_Avg_5:0": 19989.31,
|
|
"ib-loopback/IB_write_16384_Avg_5:0": 22560.57,
|
|
"ib-loopback/IB_write_32768_Avg_5:0": 23389.29,
|
|
"ib-loopback/IB_write_65536_Avg_5:0": 23503.32,
|
|
"ib-loopback/IB_write_131072_Avg_5:0": 23741.51,
|
|
"ib-loopback/IB_write_262144_Avg_5:0": 23866.43,
|
|
"ib-loopback/IB_write_524288_Avg_5:0": 23898.95,
|
|
"ib-loopback/IB_write_1048576_Avg_5:0": 23876.36,
|
|
"ib-loopback/IB_write_2097152_Avg_5:0": 23919.63,
|
|
"ib-loopback/IB_write_4194304_Avg_5:0": 23924.68,
|
|
"ib-loopback/IB_write_8388608_Avg_5:0": 23930.37,
|
|
"ib-loopback/IB_write_512_Avg_6:0": 1467.69,
|
|
"ib-loopback/IB_write_1024_Avg_6:0": 3157.04,
|
|
"ib-loopback/IB_write_2048_Avg_6:0": 6494.61,
|
|
"ib-loopback/IB_write_4096_Avg_6:0": 12883.51,
|
|
"ib-loopback/IB_write_8192_Avg_6:0": 19207.67,
|
|
"ib-loopback/IB_write_16384_Avg_6:0": 22519.39,
|
|
"ib-loopback/IB_write_32768_Avg_6:0": 23323.46,
|
|
"ib-loopback/IB_write_65536_Avg_6:0": 23523.6,
|
|
"ib-loopback/IB_write_131072_Avg_6:0": 23626.67,
|
|
"ib-loopback/IB_write_262144_Avg_6:0": 23836.99,
|
|
"ib-loopback/IB_write_524288_Avg_6:0": 23904.51,
|
|
"ib-loopback/IB_write_1048576_Avg_6:0": 23919.07,
|
|
"ib-loopback/IB_write_2097152_Avg_6:0": 23943.82,
|
|
"ib-loopback/IB_write_4194304_Avg_6:0": 23936.77,
|
|
"ib-loopback/IB_write_8388608_Avg_6:0": 23941.57,
|
|
"ib-loopback/IB_write_512_Avg_7:0": 1505.63,
|
|
"ib-loopback/IB_write_1024_Avg_7:0": 3259.93,
|
|
"ib-loopback/IB_write_2048_Avg_7:0": 6738.6,
|
|
"ib-loopback/IB_write_4096_Avg_7:0": 13352.06,
|
|
"ib-loopback/IB_write_8192_Avg_7:0": 19941.35,
|
|
"ib-loopback/IB_write_16384_Avg_7:0": 22566.09,
|
|
"ib-loopback/IB_write_32768_Avg_7:0": 23244.77,
|
|
"ib-loopback/IB_write_65536_Avg_7:0": 23377.67,
|
|
"ib-loopback/IB_write_131072_Avg_7:0": 23736.17,
|
|
"ib-loopback/IB_write_262144_Avg_7:0": 23829.25,
|
|
"ib-loopback/IB_write_524288_Avg_7:0": 23879.6,
|
|
"ib-loopback/IB_write_1048576_Avg_7:0": 23895.1,
|
|
"ib-loopback/IB_write_2097152_Avg_7:0": 23930.64,
|
|
"ib-loopback/IB_write_4194304_Avg_7:0": 23845.63,
|
|
"ib-loopback/IB_write_8388608_Avg_7:0": 23896.94,
|
|
"kernel-launch/return_code": 0,
|
|
"kernel-launch/event_overhead:0": 0.1,
|
|
"kernel-launch/event_overhead:1": 0.00595,
|
|
"kernel-launch/event_overhead:2": 0.00557,
|
|
"kernel-launch/event_overhead:3": 0.0055,
|
|
"kernel-launch/event_overhead:4": 0.00592,
|
|
"kernel-launch/event_overhead:5": 0.00589,
|
|
"kernel-launch/event_overhead:6": 0.00572,
|
|
"kernel-launch/event_overhead:7": 0.0059,
|
|
"kernel-launch/wall_overhead:0": 0.01026,
|
|
"kernel-launch/wall_overhead:1": 0.01026,
|
|
"kernel-launch/wall_overhead:2": 0.01046,
|
|
"kernel-launch/wall_overhead:3": 0.01049,
|
|
"kernel-launch/wall_overhead:4": 0.01063,
|
|
"kernel-launch/wall_overhead:5": 0.01006,
|
|
"kernel-launch/wall_overhead:6": 0.01045,
|
|
"kernel-launch/wall_overhead:7": 0.01071,
|
|
"lstm_models/pytorch-lstm/steptime_train_float32": 48.0702451896,
|
|
"lstm_models/pytorch-lstm/throughput_train_float32": 4806.4724411328,
|
|
"lstm_models/pytorch-lstm/steptime_train_float16": 25.9531298652,
|
|
"lstm_models/pytorch-lstm/throughput_train_float16": 9069.9080925588,
|
|
"pytorch-matmul/nosharding": 34.6449975967,
|
|
"mem-bw/return_code": 0,
|
|
"mem-bw/H2D_Mem_BW:0": 25.6,
|
|
"mem-bw/H2D_Mem_BW:1": 25.8,
|
|
"mem-bw/H2D_Mem_BW:2": 26,
|
|
"mem-bw/H2D_Mem_BW:3": 26.1,
|
|
"mem-bw/H2D_Mem_BW:4": 26.2,
|
|
"mem-bw/H2D_Mem_BW:5": 25.8,
|
|
"mem-bw/H2D_Mem_BW:6": 25.3,
|
|
"mem-bw/H2D_Mem_BW:7": 26.1,
|
|
"mem-bw/D2H_Mem_BW:0": 24.3,
|
|
"mem-bw/D2H_Mem_BW:1": 24.6,
|
|
"mem-bw/D2H_Mem_BW:2": 24.5,
|
|
"mem-bw/D2H_Mem_BW:3": 24.6,
|
|
"mem-bw/D2H_Mem_BW:4": 24.3,
|
|
"mem-bw/D2H_Mem_BW:5": 24.3,
|
|
"mem-bw/D2H_Mem_BW:6": 23.9,
|
|
"mem-bw/D2H_Mem_BW:7": 24.6,
|
|
"mem-bw/D2D_Mem_BW:0": 1118,
|
|
"mem-bw/D2D_Mem_BW:1": 1114.6,
|
|
"mem-bw/D2D_Mem_BW:2": 1119.7,
|
|
"mem-bw/D2D_Mem_BW:3": 1121.9,
|
|
"mem-bw/D2D_Mem_BW:4": 1109.7,
|
|
"mem-bw/D2D_Mem_BW:5": 1110.1,
|
|
"mem-bw/D2D_Mem_BW:6": 1123.3,
|
|
"mem-bw/D2D_Mem_BW:7": 1117.6,
|
|
"nccl-bw/allreduce_8_busbw:0": 0,
|
|
"nccl-bw/allreduce_8_algbw:0": 0,
|
|
"nccl-bw/allreduce_8_time:0": 37.84,
|
|
"nccl-bw/allreduce_16_busbw:0": 0,
|
|
"nccl-bw/allreduce_16_algbw:0": 0,
|
|
"nccl-bw/allreduce_16_time:0": 36.42,
|
|
"nccl-bw/allreduce_32_busbw:0": 0,
|
|
"nccl-bw/allreduce_32_algbw:0": 0,
|
|
"nccl-bw/allreduce_32_time:0": 36.87,
|
|
"nccl-bw/allreduce_64_busbw:0": 0,
|
|
"nccl-bw/allreduce_64_algbw:0": 0,
|
|
"nccl-bw/allreduce_64_time:0": 35.83,
|
|
"nccl-bw/allreduce_128_busbw:0": 0.01,
|
|
"nccl-bw/allreduce_128_algbw:0": 0,
|
|
"nccl-bw/allreduce_128_time:0": 36.91,
|
|
"nccl-bw/allreduce_256_busbw:0": 0.01,
|
|
"nccl-bw/allreduce_256_algbw:0": 0.01,
|
|
"nccl-bw/allreduce_256_time:0": 37.58,
|
|
"nccl-bw/allreduce_512_busbw:0": 0.02,
|
|
"nccl-bw/allreduce_512_algbw:0": 0.01,
|
|
"nccl-bw/allreduce_512_time:0": 36.98,
|
|
"nccl-bw/allreduce_1024_busbw:0": 0.05,
|
|
"nccl-bw/allreduce_1024_algbw:0": 0.03,
|
|
"nccl-bw/allreduce_1024_time:0": 36.93,
|
|
"nccl-bw/allreduce_2048_busbw:0": 0.1,
|
|
"nccl-bw/allreduce_2048_algbw:0": 0.06,
|
|
"nccl-bw/allreduce_2048_time:0": 36.06,
|
|
"nccl-bw/allreduce_4096_busbw:0": 0.19,
|
|
"nccl-bw/allreduce_4096_algbw:0": 0.11,
|
|
"nccl-bw/allreduce_4096_time:0": 37.2,
|
|
"nccl-bw/allreduce_8192_busbw:0": 0.39,
|
|
"nccl-bw/allreduce_8192_algbw:0": 0.22,
|
|
"nccl-bw/allreduce_8192_time:0": 37.04,
|
|
"nccl-bw/allreduce_16384_busbw:0": 0.77,
|
|
"nccl-bw/allreduce_16384_algbw:0": 0.44,
|
|
"nccl-bw/allreduce_16384_time:0": 37.46,
|
|
"nccl-bw/allreduce_32768_busbw:0": 1.52,
|
|
"nccl-bw/allreduce_32768_algbw:0": 0.87,
|
|
"nccl-bw/allreduce_32768_time:0": 37.64,
|
|
"nccl-bw/allreduce_65536_busbw:0": 3,
|
|
"nccl-bw/allreduce_65536_algbw:0": 1.71,
|
|
"nccl-bw/allreduce_65536_time:0": 38.22,
|
|
"nccl-bw/allreduce_131072_busbw:0": 5.31,
|
|
"nccl-bw/allreduce_131072_algbw:0": 3.04,
|
|
"nccl-bw/allreduce_131072_time:0": 43.17,
|
|
"nccl-bw/allreduce_262144_busbw:0": 9.5,
|
|
"nccl-bw/allreduce_262144_algbw:0": 5.43,
|
|
"nccl-bw/allreduce_262144_time:0": 48.29,
|
|
"nccl-bw/allreduce_524288_busbw:0": 15.11,
|
|
"nccl-bw/allreduce_524288_algbw:0": 8.64,
|
|
"nccl-bw/allreduce_524288_time:0": 60.71,
|
|
"nccl-bw/allreduce_1048576_busbw:0": 24.1,
|
|
"nccl-bw/allreduce_1048576_algbw:0": 13.77,
|
|
"nccl-bw/allreduce_1048576_time:0": 76.13,
|
|
"nccl-bw/allreduce_2097152_busbw:0": 38.12,
|
|
"nccl-bw/allreduce_2097152_algbw:0": 21.78,
|
|
"nccl-bw/allreduce_2097152_time:0": 96.28,
|
|
"nccl-bw/allreduce_4194304_busbw:0": 65.75,
|
|
"nccl-bw/allreduce_4194304_algbw:0": 37.57,
|
|
"nccl-bw/allreduce_4194304_time:0": 111.6,
|
|
"nccl-bw/allreduce_8388608_busbw:0": 89.51,
|
|
"nccl-bw/allreduce_8388608_algbw:0": 51.15,
|
|
"nccl-bw/allreduce_8388608_time:0": 164,
|
|
"nccl-bw/allreduce_16777216_busbw:0": 114.38,
|
|
"nccl-bw/allreduce_16777216_algbw:0": 65.36,
|
|
"nccl-bw/allreduce_16777216_time:0": 256.7,
|
|
"nccl-bw/allreduce_33554432_busbw:0": 154.89,
|
|
"nccl-bw/allreduce_33554432_algbw:0": 88.51,
|
|
"nccl-bw/allreduce_33554432_time:0": 379.1,
|
|
"nccl-bw/allreduce_67108864_busbw:0": 200.01,
|
|
"nccl-bw/allreduce_67108864_algbw:0": 114.29,
|
|
"nccl-bw/allreduce_67108864_time:0": 587.2,
|
|
"nccl-bw/allreduce_134217728_busbw:0": 202.97,
|
|
"nccl-bw/allreduce_134217728_algbw:0": 115.98,
|
|
"nccl-bw/allreduce_134217728_time:0": 1157.2,
|
|
"nccl-bw/allreduce_268435456_busbw:0": 221.82,
|
|
"nccl-bw/allreduce_268435456_algbw:0": 126.75,
|
|
"nccl-bw/allreduce_268435456_time:0": 2117.8,
|
|
"nccl-bw/allreduce_536870912_busbw:0": 224.54,
|
|
"nccl-bw/allreduce_536870912_algbw:0": 128.31,
|
|
"nccl-bw/allreduce_536870912_time:0": 4184.2,
|
|
"nccl-bw/allreduce_1073741824_busbw:0": 230.15,
|
|
"nccl-bw/allreduce_1073741824_algbw:0": 131.51,
|
|
"nccl-bw/allreduce_1073741824_time:0": 8164.5,
|
|
"nccl-bw/allreduce_2147483648_busbw:0": 231.89,
|
|
"nccl-bw/allreduce_2147483648_algbw:0": 132.51,
|
|
"nccl-bw/allreduce_2147483648_time:0": 16207,
|
|
"nccl-bw/allreduce_4294967296_busbw:0": 234.45,
|
|
"nccl-bw/allreduce_4294967296_algbw:0": 133.97,
|
|
"nccl-bw/allreduce_4294967296_time:0": 32059,
|
|
"nccl-bw/allreduce_8589934592_busbw:0": 235.36,
|
|
"nccl-bw/allreduce_8589934592_algbw:0": 134.49,
|
|
"nccl-bw/allreduce_8589934592_time:0": 63870,
|
|
"resnet_models/pytorch-resnet50/steptime_train_float32": 253.9552273229,
|
|
"resnet_models/pytorch-resnet50/throughput_train_float32": 760.334809913,
|
|
"resnet_models/pytorch-resnet50/steptime_train_float16": 200.0860618427,
|
|
"resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430923,
|
|
"resnet_models/pytorch-resnet101/steptime_train_float32": 389.0860509127,
|
|
"resnet_models/pytorch-resnet101/throughput_train_float32": 496.117474093,
|
|
"resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918,
|
|
"resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195,
|
|
"resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278315,
|
|
"resnet_models/pytorch-resnet152/throughput_train_float32": 352.0709954335,
|
|
"resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802,
|
|
"resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998154,
|
|
"pytorch-sharding-matmul/allreduce": 10.574411869,
|
|
"pytorch-sharding-matmul/allgather": 10.0846967697,
|
|
"vgg_models/pytorch-vgg11/steptime_train_float32": 40.3528367169,
|
|
"vgg_models/pytorch-vgg11/throughput_train_float32": 796.361593695,
|
|
"vgg_models/pytorch-vgg11/steptime_train_float16": 24.1335148457,
|
|
"vgg_models/pytorch-vgg11/throughput_train_float16": 1330.4113614585,
|
|
"vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936,
|
|
"vgg_models/pytorch-vgg13/throughput_train_float32": 580.2341074444,
|
|
"vgg_models/pytorch-vgg13/steptime_train_float16": 33.3522899309,
|
|
"vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023902,
|
|
"vgg_models/pytorch-vgg16/steptime_train_float32": 65.225199447,
|
|
"vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638876,
|
|
"vgg_models/pytorch-vgg16/steptime_train_float16": 39.2528773518,
|
|
"vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546148,
|
|
"vgg_models/pytorch-vgg19/steptime_train_float32": 74.9348710524,
|
|
"vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
|
|
"vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
|
|
"vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377
|
|
},
|
|
{
|
|
"index": "sb-validation-02",
|
|
"diagnosis/accept": true,
|
|
"diagnosis/issue_num": 0,
|
|
"diagnosis/category": "N/A",
|
|
"diagnosis/issue_details": "N/A",
|
|
"bert_models/pytorch-bert-base/steptime_train_float32": 114.5916701062,
|
|
"bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591,
|
|
"bert_models/pytorch-bert-base/steptime_train_float16": 83.8895108318,
|
|
"bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582742,
|
|
"bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371914,
|
|
"bert_models/pytorch-bert-large/throughput_train_float32": 103.9487609742,
|
|
"bert_models/pytorch-bert-large/steptime_train_float16": 206.8114168942,
|
|
"bert_models/pytorch-bert-large/throughput_train_float16": 154.8408911711,
|
|
"pytorch-computation-communication-overlap/mul_cost:0": 44.1822062144,
|
|
"pytorch-computation-communication-overlap/mul_cost:1": 44.1822139389,
|
|
"pytorch-computation-communication-overlap/mul_cost:2": 43.9701470781,
|
|
"pytorch-computation-communication-overlap/mul_cost:3": 43.9701478756,
|
|
"pytorch-computation-communication-overlap/mul_cost:4": 43.9701779317,
|
|
"pytorch-computation-communication-overlap/mul_cost:5": 43.9701571606,
|
|
"pytorch-computation-communication-overlap/mul_cost:6": 43.9701651983,
|
|
"pytorch-computation-communication-overlap/mul_cost:7": 44.1795444785,
|
|
"pytorch-computation-communication-overlap/matmul_cost:0": 137.0477370556,
|
|
"pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344693,
|
|
"pytorch-computation-communication-overlap/matmul_cost:2": 137.047772209,
|
|
"pytorch-computation-communication-overlap/matmul_cost:3": 137.0477969726,
|
|
"pytorch-computation-communication-overlap/matmul_cost:4": 137.0481367431,
|
|
"pytorch-computation-communication-overlap/matmul_cost:5": 137.0482198877,
|
|
"pytorch-computation-communication-overlap/matmul_cost:6": 137.0477532237,
|
|
"pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081607,
|
|
"densenet_models/pytorch-densenet169/steptime_train_float32": 150.6415554322,
|
|
"densenet_models/pytorch-densenet169/throughput_train_float32": 212.4701719243,
|
|
"densenet_models/pytorch-densenet169/steptime_train_float16": 145.3081957297,
|
|
"densenet_models/pytorch-densenet169/throughput_train_float16": 220.2932337305,
|
|
"densenet_models/pytorch-densenet201/steptime_train_float32": 182.9111778643,
|
|
"densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232,
|
|
"densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596,
|
|
"densenet_models/pytorch-densenet201/throughput_train_float16": 181.5446573603,
|
|
"gemm-flops/FP64:0": 9031.23,
|
|
"gemm-flops/FP64:1": 9040.85,
|
|
"gemm-flops/FP64:2": 9010.56,
|
|
"gemm-flops/FP64:3": 9041.26,
|
|
"gemm-flops/FP64:4": 9039.19,
|
|
"gemm-flops/FP64:5": 9015.69,
|
|
"gemm-flops/FP64:6": 9022.19,
|
|
"gemm-flops/FP64:7": 9030.2,
|
|
"gemm-flops/FP32:0": 18362.1,
|
|
"gemm-flops/FP32:1": 18375.6,
|
|
"gemm-flops/FP32:2": 18314.9,
|
|
"gemm-flops/FP32:3": 18375.6,
|
|
"gemm-flops/FP32:4": 18368.6,
|
|
"gemm-flops/FP32:5": 18347.1,
|
|
"gemm-flops/FP32:6": 18247.4,
|
|
"gemm-flops/FP32:7": 18318.4,
|
|
"gemm-flops/FP16:0": 33878,
|
|
"gemm-flops/FP16:1": 33911.1,
|
|
"gemm-flops/FP16:2": 33769.3,
|
|
"gemm-flops/FP16:3": 33909.9,
|
|
"gemm-flops/FP16:4": 33896.5,
|
|
"gemm-flops/FP16:5": 33798.1,
|
|
"gemm-flops/FP16:6": 33647.3,
|
|
"gemm-flops/FP16:7": 33764.8,
|
|
"gemm-flops/FP64_TC:0": 18963.6,
|
|
"gemm-flops/FP64_TC:1": 18924.2,
|
|
"gemm-flops/FP64_TC:2": 18930.3,
|
|
"gemm-flops/FP64_TC:3": 18971.9,
|
|
"gemm-flops/FP64_TC:4": 18946,
|
|
"gemm-flops/FP64_TC:5": 18945,
|
|
"gemm-flops/FP64_TC:6": 18822.9,
|
|
"gemm-flops/FP64_TC:7": 18911.1,
|
|
"gemm-flops/TF32_TC:0": 127900,
|
|
"gemm-flops/TF32_TC:1": 129094,
|
|
"gemm-flops/TF32_TC:2": 127831,
|
|
"gemm-flops/TF32_TC:3": 128709,
|
|
"gemm-flops/TF32_TC:4": 127388,
|
|
"gemm-flops/TF32_TC:5": 127861,
|
|
"gemm-flops/TF32_TC:6": 128492,
|
|
"gemm-flops/TF32_TC:7": 127720,
|
|
"gemm-flops/BF16_TC:0": 264965,
|
|
"gemm-flops/BF16_TC:1": 266638,
|
|
"gemm-flops/BF16_TC:2": 263151,
|
|
"gemm-flops/BF16_TC:3": 264752,
|
|
"gemm-flops/BF16_TC:4": 263049,
|
|
"gemm-flops/BF16_TC:5": 266605,
|
|
"gemm-flops/BF16_TC:6": 267501,
|
|
"gemm-flops/BF16_TC:7": 263880,
|
|
"gemm-flops/FP16_TC:0": 279474,
|
|
"gemm-flops/FP16_TC:1": 281256,
|
|
"gemm-flops/FP16_TC:2": 277403,
|
|
"gemm-flops/FP16_TC:3": 279147,
|
|
"gemm-flops/FP16_TC:4": 277587,
|
|
"gemm-flops/FP16_TC:5": 281537,
|
|
"gemm-flops/FP16_TC:6": 282132,
|
|
"gemm-flops/FP16_TC:7": 277788,
|
|
"gemm-flops/INT8_TC:0": 475160,
|
|
"gemm-flops/INT8_TC:1": 477725,
|
|
"gemm-flops/INT8_TC:2": 471621,
|
|
"gemm-flops/INT8_TC:3": 473716,
|
|
"gemm-flops/INT8_TC:4": 472124,
|
|
"gemm-flops/INT8_TC:5": 479972,
|
|
"gemm-flops/INT8_TC:6": 481327,
|
|
"gemm-flops/INT8_TC:7": 474710,
|
|
"gemm-flops/INT4_TC:0": 970330,
|
|
"gemm-flops/INT4_TC:1": 976837,
|
|
"gemm-flops/INT4_TC:2": 966003,
|
|
"gemm-flops/INT4_TC:3": 971315,
|
|
"gemm-flops/INT4_TC:4": 964441,
|
|
"gemm-flops/INT4_TC:5": 982461,
|
|
"gemm-flops/INT4_TC:6": 979610,
|
|
"gemm-flops/INT4_TC:7": 968359,
|
|
"gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971836,
|
|
"gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.1154543969,
|
|
"gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742235,
|
|
"gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.1394499411,
|
|
"gpu-sm-copy-bw/dtoh:0": 3.91755,
|
|
"gpu-sm-copy-bw/dtoh:1": 4.45414,
|
|
"gpu-sm-copy-bw/dtoh:2": 1.26483,
|
|
"gpu-sm-copy-bw/dtoh:3": 1.30041,
|
|
"gpu-sm-copy-bw/dtoh:4": 1.31577,
|
|
"gpu-sm-copy-bw/dtoh:5": 1.27968,
|
|
"gpu-sm-copy-bw/dtoh:6": 4.47849,
|
|
"gpu-sm-copy-bw/dtoh:7": 3.96231,
|
|
"gpu-sm-copy-bw/dtoh:8": 3.91705,
|
|
"gpu-sm-copy-bw/dtoh:9": 4.45487,
|
|
"gpu-sm-copy-bw/dtoh:10": 1.26352,
|
|
"gpu-sm-copy-bw/dtoh:11": 1.2999,
|
|
"gpu-sm-copy-bw/dtoh:12": 1.31677,
|
|
"gpu-sm-copy-bw/dtoh:13": 1.27885,
|
|
"gpu-sm-copy-bw/dtoh:14": 4.47913,
|
|
"gpu-sm-copy-bw/dtoh:15": 3.95893,
|
|
"gpu-sm-copy-bw/dtoh:16": 3.91729,
|
|
"gpu-sm-copy-bw/dtoh:17": 4.45627,
|
|
"gpu-sm-copy-bw/dtoh:18": 1.26437,
|
|
"gpu-sm-copy-bw/dtoh:19": 1.30144,
|
|
"gpu-sm-copy-bw/dtoh:20": 1.31704,
|
|
"gpu-sm-copy-bw/dtoh:21": 1.27857,
|
|
"gpu-sm-copy-bw/dtoh:22": 4.47889,
|
|
"gpu-sm-copy-bw/dtoh:23": 3.95984,
|
|
"gpu-sm-copy-bw/dtoh:24": 3.92025,
|
|
"gpu-sm-copy-bw/dtoh:25": 4.45423,
|
|
"gpu-sm-copy-bw/dtoh:26": 1.26449,
|
|
"gpu-sm-copy-bw/dtoh:27": 1.29954,
|
|
"gpu-sm-copy-bw/dtoh:28": 1.31731,
|
|
"gpu-sm-copy-bw/dtoh:29": 1.27916,
|
|
"gpu-sm-copy-bw/dtoh:30": 4.4797,
|
|
"gpu-sm-copy-bw/dtoh:31": 3.96124,
|
|
"gpu-sm-copy-bw/htod:0": 23.9685,
|
|
"gpu-sm-copy-bw/htod:1": 23.967,
|
|
"gpu-sm-copy-bw/htod:2": 19.9898,
|
|
"gpu-sm-copy-bw/htod:3": 20.2848,
|
|
"gpu-sm-copy-bw/htod:4": 20.3931,
|
|
"gpu-sm-copy-bw/htod:5": 20.0888,
|
|
"gpu-sm-copy-bw/htod:6": 23.9766,
|
|
"gpu-sm-copy-bw/htod:7": 23.9792,
|
|
"gpu-sm-copy-bw/htod:8": 23.9497,
|
|
"gpu-sm-copy-bw/htod:9": 23.9438,
|
|
"gpu-sm-copy-bw/htod:10": 20.0039,
|
|
"gpu-sm-copy-bw/htod:11": 20.2469,
|
|
"gpu-sm-copy-bw/htod:12": 20.4344,
|
|
"gpu-sm-copy-bw/htod:13": 20.1005,
|
|
"gpu-sm-copy-bw/htod:14": 23.9491,
|
|
"gpu-sm-copy-bw/htod:15": 23.9898,
|
|
"gpu-sm-copy-bw/htod:16": 23.962,
|
|
"gpu-sm-copy-bw/htod:17": 23.97,
|
|
"gpu-sm-copy-bw/htod:18": 19.987,
|
|
"gpu-sm-copy-bw/htod:19": 20.1549,
|
|
"gpu-sm-copy-bw/htod:20": 20.3931,
|
|
"gpu-sm-copy-bw/htod:21": 20.1013,
|
|
"gpu-sm-copy-bw/htod:22": 23.9865,
|
|
"gpu-sm-copy-bw/htod:23": 23.972,
|
|
"gpu-sm-copy-bw/htod:24": 23.9569,
|
|
"gpu-sm-copy-bw/htod:25": 23.9762,
|
|
"gpu-sm-copy-bw/htod:26": 19.92,
|
|
"gpu-sm-copy-bw/htod:27": 20.2638,
|
|
"gpu-sm-copy-bw/htod:28": 20.4419,
|
|
"gpu-sm-copy-bw/htod:29": 20.1054,
|
|
"gpu-sm-copy-bw/htod:30": 23.9752,
|
|
"gpu-sm-copy-bw/htod:31": 23.9631,
|
|
"ib-loopback/IB_write_512_Avg_0:0": 1492.89,
|
|
"ib-loopback/IB_write_1024_Avg_0:0": 3224.92,
|
|
"ib-loopback/IB_write_2048_Avg_0:0": 6714.5,
|
|
"ib-loopback/IB_write_4096_Avg_0:0": 12871.93,
|
|
"ib-loopback/IB_write_8192_Avg_0:0": 19990.78,
|
|
"ib-loopback/IB_write_16384_Avg_0:0": 22172.25,
|
|
"ib-loopback/IB_write_32768_Avg_0:0": 23073.19,
|
|
"ib-loopback/IB_write_65536_Avg_0:0": 23527.09,
|
|
"ib-loopback/IB_write_131072_Avg_0:0": 23805.92,
|
|
"ib-loopback/IB_write_262144_Avg_0:0": 23380.51,
|
|
"ib-loopback/IB_write_524288_Avg_0:0": 23856.63,
|
|
"ib-loopback/IB_write_1048576_Avg_0:0": 23869.58,
|
|
"ib-loopback/IB_write_2097152_Avg_0:0": 23885.7,
|
|
"ib-loopback/IB_write_4194304_Avg_0:0": 23914.56,
|
|
"ib-loopback/IB_write_8388608_Avg_0:0": 23935.21,
|
|
"ib-loopback/IB_write_512_Avg_1:0": 1479.1,
|
|
"ib-loopback/IB_write_1024_Avg_1:0": 3246.48,
|
|
"ib-loopback/IB_write_2048_Avg_1:0": 6754.94,
|
|
"ib-loopback/IB_write_4096_Avg_1:0": 13101.09,
|
|
"ib-loopback/IB_write_8192_Avg_1:0": 19945.81,
|
|
"ib-loopback/IB_write_16384_Avg_1:0": 22631.62,
|
|
"ib-loopback/IB_write_32768_Avg_1:0": 23381.58,
|
|
"ib-loopback/IB_write_65536_Avg_1:0": 23515.29,
|
|
"ib-loopback/IB_write_131072_Avg_1:0": 23756.01,
|
|
"ib-loopback/IB_write_262144_Avg_1:0": 23875.91,
|
|
"ib-loopback/IB_write_524288_Avg_1:0": 23911.13,
|
|
"ib-loopback/IB_write_1048576_Avg_1:0": 23935.42,
|
|
"ib-loopback/IB_write_2097152_Avg_1:0": 23941.25,
|
|
"ib-loopback/IB_write_4194304_Avg_1:0": 23922.06,
|
|
"ib-loopback/IB_write_8388608_Avg_1:0": 23928.59,
|
|
"ib-loopback/IB_write_512_Avg_2:0": 1505.29,
|
|
"ib-loopback/IB_write_1024_Avg_2:0": 3215.97,
|
|
"ib-loopback/IB_write_2048_Avg_2:0": 6745.49,
|
|
"ib-loopback/IB_write_4096_Avg_2:0": 11548.88,
|
|
"ib-loopback/IB_write_8192_Avg_2:0": 19432.15,
|
|
"ib-loopback/IB_write_16384_Avg_2:0": 22765.51,
|
|
"ib-loopback/IB_write_32768_Avg_2:0": 23235.07,
|
|
"ib-loopback/IB_write_65536_Avg_2:0": 23620.08,
|
|
"ib-loopback/IB_write_131072_Avg_2:0": 23759.08,
|
|
"ib-loopback/IB_write_262144_Avg_2:0": 23859.82,
|
|
"ib-loopback/IB_write_524288_Avg_2:0": 23775.01,
|
|
"ib-loopback/IB_write_1048576_Avg_2:0": 23885.96,
|
|
"ib-loopback/IB_write_2097152_Avg_2:0": 23894.73,
|
|
"ib-loopback/IB_write_4194304_Avg_2:0": 23909.98,
|
|
"ib-loopback/IB_write_8388608_Avg_2:0": 23927.21,
|
|
"ib-loopback/IB_write_512_Avg_3:0": 1474.47,
|
|
"ib-loopback/IB_write_1024_Avg_3:0": 3344.68,
|
|
"ib-loopback/IB_write_2048_Avg_3:0": 6606.35,
|
|
"ib-loopback/IB_write_4096_Avg_3:0": 12071.4,
|
|
"ib-loopback/IB_write_8192_Avg_3:0": 18408.33,
|
|
"ib-loopback/IB_write_16384_Avg_3:0": 20789.27,
|
|
"ib-loopback/IB_write_32768_Avg_3:0": 22469.43,
|
|
"ib-loopback/IB_write_65536_Avg_3:0": 22777.1,
|
|
"ib-loopback/IB_write_131072_Avg_3:0": 23461.03,
|
|
"ib-loopback/IB_write_262144_Avg_3:0": 23397.19,
|
|
"ib-loopback/IB_write_524288_Avg_3:0": 23526.55,
|
|
"ib-loopback/IB_write_1048576_Avg_3:0": 23854.76,
|
|
"ib-loopback/IB_write_2097152_Avg_3:0": 23862.23,
|
|
"ib-loopback/IB_write_4194304_Avg_3:0": 23931.15,
|
|
"ib-loopback/IB_write_8388608_Avg_3:0": 23924.44,
|
|
"ib-loopback/IB_write_512_Avg_4:0": 1523.33,
|
|
"ib-loopback/IB_write_1024_Avg_4:0": 3233.23,
|
|
"ib-loopback/IB_write_2048_Avg_4:0": 6792.88,
|
|
"ib-loopback/IB_write_4096_Avg_4:0": 12616.05,
|
|
"ib-loopback/IB_write_8192_Avg_4:0": 19324.05,
|
|
"ib-loopback/IB_write_16384_Avg_4:0": 22082.51,
|
|
"ib-loopback/IB_write_32768_Avg_4:0": 23294.23,
|
|
"ib-loopback/IB_write_65536_Avg_4:0": 23546.22,
|
|
"ib-loopback/IB_write_131072_Avg_4:0": 23727.91,
|
|
"ib-loopback/IB_write_262144_Avg_4:0": 23843.93,
|
|
"ib-loopback/IB_write_524288_Avg_4:0": 23905.96,
|
|
"ib-loopback/IB_write_1048576_Avg_4:0": 23902.37,
|
|
"ib-loopback/IB_write_2097152_Avg_4:0": 23921.03,
|
|
"ib-loopback/IB_write_4194304_Avg_4:0": 23921.06,
|
|
"ib-loopback/IB_write_8388608_Avg_4:0": 23922.4,
|
|
"ib-loopback/IB_write_512_Avg_5:0": 1506.63,
|
|
"ib-loopback/IB_write_1024_Avg_5:0": 3261.71,
|
|
"ib-loopback/IB_write_2048_Avg_5:0": 6752.38,
|
|
"ib-loopback/IB_write_4096_Avg_5:0": 13592.41,
|
|
"ib-loopback/IB_write_8192_Avg_5:0": 19989.31,
|
|
"ib-loopback/IB_write_16384_Avg_5:0": 22560.57,
|
|
"ib-loopback/IB_write_32768_Avg_5:0": 23389.29,
|
|
"ib-loopback/IB_write_65536_Avg_5:0": 23503.32,
|
|
"ib-loopback/IB_write_131072_Avg_5:0": 23741.51,
|
|
"ib-loopback/IB_write_262144_Avg_5:0": 23866.43,
|
|
"ib-loopback/IB_write_524288_Avg_5:0": 23898.95,
|
|
"ib-loopback/IB_write_1048576_Avg_5:0": 23876.36,
|
|
"ib-loopback/IB_write_2097152_Avg_5:0": 23919.63,
|
|
"ib-loopback/IB_write_4194304_Avg_5:0": 23924.68,
|
|
"ib-loopback/IB_write_8388608_Avg_5:0": 23930.37,
|
|
"ib-loopback/IB_write_512_Avg_6:0": 1467.69,
|
|
"ib-loopback/IB_write_1024_Avg_6:0": 3157.04,
|
|
"ib-loopback/IB_write_2048_Avg_6:0": 6494.61,
|
|
"ib-loopback/IB_write_4096_Avg_6:0": 12883.51,
|
|
"ib-loopback/IB_write_8192_Avg_6:0": 19207.67,
|
|
"ib-loopback/IB_write_16384_Avg_6:0": 22519.39,
|
|
"ib-loopback/IB_write_32768_Avg_6:0": 23323.46,
|
|
"ib-loopback/IB_write_65536_Avg_6:0": 23523.6,
|
|
"ib-loopback/IB_write_131072_Avg_6:0": 23626.67,
|
|
"ib-loopback/IB_write_262144_Avg_6:0": 23836.99,
|
|
"ib-loopback/IB_write_524288_Avg_6:0": 23904.51,
|
|
"ib-loopback/IB_write_1048576_Avg_6:0": 23919.07,
|
|
"ib-loopback/IB_write_2097152_Avg_6:0": 23943.82,
|
|
"ib-loopback/IB_write_4194304_Avg_6:0": 23936.77,
|
|
"ib-loopback/IB_write_8388608_Avg_6:0": 23941.57,
|
|
"ib-loopback/IB_write_512_Avg_7:0": 1505.63,
|
|
"ib-loopback/IB_write_1024_Avg_7:0": 3259.93,
|
|
"ib-loopback/IB_write_2048_Avg_7:0": 6738.6,
|
|
"ib-loopback/IB_write_4096_Avg_7:0": 13352.06,
|
|
"ib-loopback/IB_write_8192_Avg_7:0": 19941.35,
|
|
"ib-loopback/IB_write_16384_Avg_7:0": 22566.09,
|
|
"ib-loopback/IB_write_32768_Avg_7:0": 23244.77,
|
|
"ib-loopback/IB_write_65536_Avg_7:0": 23377.67,
|
|
"ib-loopback/IB_write_131072_Avg_7:0": 23736.17,
|
|
"ib-loopback/IB_write_262144_Avg_7:0": 23829.25,
|
|
"ib-loopback/IB_write_524288_Avg_7:0": 23879.6,
|
|
"ib-loopback/IB_write_1048576_Avg_7:0": 23895.1,
|
|
"ib-loopback/IB_write_2097152_Avg_7:0": 23930.64,
|
|
"ib-loopback/IB_write_4194304_Avg_7:0": 23845.63,
|
|
"ib-loopback/IB_write_8388608_Avg_7:0": 23896.94,
|
|
"kernel-launch/return_code": 0,
|
|
"kernel-launch/event_overhead:0": 0.00595,
|
|
"kernel-launch/event_overhead:1": 0.00595,
|
|
"kernel-launch/event_overhead:2": 0.00557,
|
|
"kernel-launch/event_overhead:3": 0.0055,
|
|
"kernel-launch/event_overhead:4": 0.00592,
|
|
"kernel-launch/event_overhead:5": 0.00589,
|
|
"kernel-launch/event_overhead:6": 0.00572,
|
|
"kernel-launch/event_overhead:7": 0.0059,
|
|
"kernel-launch/wall_overhead:0": 0.01026,
|
|
"kernel-launch/wall_overhead:1": 0.01026,
|
|
"kernel-launch/wall_overhead:2": 0.01046,
|
|
"kernel-launch/wall_overhead:3": 0.01049,
|
|
"kernel-launch/wall_overhead:4": 0.01063,
|
|
"kernel-launch/wall_overhead:5": 0.01006,
|
|
"kernel-launch/wall_overhead:6": 0.01045,
|
|
"kernel-launch/wall_overhead:7": 0.01071,
|
|
"lstm_models/pytorch-lstm/steptime_train_float32": 48.0702451896,
|
|
"lstm_models/pytorch-lstm/throughput_train_float32": 4806.4724411328,
|
|
"lstm_models/pytorch-lstm/steptime_train_float16": 25.9531298652,
|
|
"lstm_models/pytorch-lstm/throughput_train_float16": 9069.9080925588,
|
|
"pytorch-matmul/nosharding": 34.6449975967,
|
|
"mem-bw/return_code": 0,
|
|
"mem-bw/H2D_Mem_BW:0": 25.6,
|
|
"mem-bw/H2D_Mem_BW:1": 25.8,
|
|
"mem-bw/H2D_Mem_BW:2": 26,
|
|
"mem-bw/H2D_Mem_BW:3": 26.1,
|
|
"mem-bw/H2D_Mem_BW:4": 26.2,
|
|
"mem-bw/H2D_Mem_BW:5": 25.8,
|
|
"mem-bw/H2D_Mem_BW:6": 25.3,
|
|
"mem-bw/H2D_Mem_BW:7": 26.1,
|
|
"mem-bw/D2H_Mem_BW:0": 24.3,
|
|
"mem-bw/D2H_Mem_BW:1": 24.6,
|
|
"mem-bw/D2H_Mem_BW:2": 24.5,
|
|
"mem-bw/D2H_Mem_BW:3": 24.6,
|
|
"mem-bw/D2H_Mem_BW:4": 24.3,
|
|
"mem-bw/D2H_Mem_BW:5": 24.3,
|
|
"mem-bw/D2H_Mem_BW:6": 23.9,
|
|
"mem-bw/D2H_Mem_BW:7": 24.6,
|
|
"mem-bw/D2D_Mem_BW:0": 1118,
|
|
"mem-bw/D2D_Mem_BW:1": 1114.6,
|
|
"mem-bw/D2D_Mem_BW:2": 1119.7,
|
|
"mem-bw/D2D_Mem_BW:3": 1121.9,
|
|
"mem-bw/D2D_Mem_BW:4": 1109.7,
|
|
"mem-bw/D2D_Mem_BW:5": 1110.1,
|
|
"mem-bw/D2D_Mem_BW:6": 1123.3,
|
|
"mem-bw/D2D_Mem_BW:7": 1117.6,
|
|
"nccl-bw/allreduce_8_busbw:0": 0,
|
|
"nccl-bw/allreduce_8_algbw:0": 0,
|
|
"nccl-bw/allreduce_8_time:0": 37.84,
|
|
"nccl-bw/allreduce_16_busbw:0": 0,
|
|
"nccl-bw/allreduce_16_algbw:0": 0,
|
|
"nccl-bw/allreduce_16_time:0": 36.42,
|
|
"nccl-bw/allreduce_32_busbw:0": 0,
|
|
"nccl-bw/allreduce_32_algbw:0": 0,
|
|
"nccl-bw/allreduce_32_time:0": 36.87,
|
|
"nccl-bw/allreduce_64_busbw:0": 0,
|
|
"nccl-bw/allreduce_64_algbw:0": 0,
|
|
"nccl-bw/allreduce_64_time:0": 35.83,
|
|
"nccl-bw/allreduce_128_busbw:0": 0.01,
|
|
"nccl-bw/allreduce_128_algbw:0": 0,
|
|
"nccl-bw/allreduce_128_time:0": 36.91,
|
|
"nccl-bw/allreduce_256_busbw:0": 0.01,
|
|
"nccl-bw/allreduce_256_algbw:0": 0.01,
|
|
"nccl-bw/allreduce_256_time:0": 37.58,
|
|
"nccl-bw/allreduce_512_busbw:0": 0.02,
|
|
"nccl-bw/allreduce_512_algbw:0": 0.01,
|
|
"nccl-bw/allreduce_512_time:0": 36.98,
|
|
"nccl-bw/allreduce_1024_busbw:0": 0.05,
|
|
"nccl-bw/allreduce_1024_algbw:0": 0.03,
|
|
"nccl-bw/allreduce_1024_time:0": 36.93,
|
|
"nccl-bw/allreduce_2048_busbw:0": 0.1,
|
|
"nccl-bw/allreduce_2048_algbw:0": 0.06,
|
|
"nccl-bw/allreduce_2048_time:0": 36.06,
|
|
"nccl-bw/allreduce_4096_busbw:0": 0.19,
|
|
"nccl-bw/allreduce_4096_algbw:0": 0.11,
|
|
"nccl-bw/allreduce_4096_time:0": 37.2,
|
|
"nccl-bw/allreduce_8192_busbw:0": 0.39,
|
|
"nccl-bw/allreduce_8192_algbw:0": 0.22,
|
|
"nccl-bw/allreduce_8192_time:0": 37.04,
|
|
"nccl-bw/allreduce_16384_busbw:0": 0.77,
|
|
"nccl-bw/allreduce_16384_algbw:0": 0.44,
|
|
"nccl-bw/allreduce_16384_time:0": 37.46,
|
|
"nccl-bw/allreduce_32768_busbw:0": 1.52,
|
|
"nccl-bw/allreduce_32768_algbw:0": 0.87,
|
|
"nccl-bw/allreduce_32768_time:0": 37.64,
|
|
"nccl-bw/allreduce_65536_busbw:0": 3,
|
|
"nccl-bw/allreduce_65536_algbw:0": 1.71,
|
|
"nccl-bw/allreduce_65536_time:0": 38.22,
|
|
"nccl-bw/allreduce_131072_busbw:0": 5.31,
|
|
"nccl-bw/allreduce_131072_algbw:0": 3.04,
|
|
"nccl-bw/allreduce_131072_time:0": 43.17,
|
|
"nccl-bw/allreduce_262144_busbw:0": 9.5,
|
|
"nccl-bw/allreduce_262144_algbw:0": 5.43,
|
|
"nccl-bw/allreduce_262144_time:0": 48.29,
|
|
"nccl-bw/allreduce_524288_busbw:0": 15.11,
|
|
"nccl-bw/allreduce_524288_algbw:0": 8.64,
|
|
"nccl-bw/allreduce_524288_time:0": 60.71,
|
|
"nccl-bw/allreduce_1048576_busbw:0": 24.1,
|
|
"nccl-bw/allreduce_1048576_algbw:0": 13.77,
|
|
"nccl-bw/allreduce_1048576_time:0": 76.13,
|
|
"nccl-bw/allreduce_2097152_busbw:0": 38.12,
|
|
"nccl-bw/allreduce_2097152_algbw:0": 21.78,
|
|
"nccl-bw/allreduce_2097152_time:0": 96.28,
|
|
"nccl-bw/allreduce_4194304_busbw:0": 65.75,
|
|
"nccl-bw/allreduce_4194304_algbw:0": 37.57,
|
|
"nccl-bw/allreduce_4194304_time:0": 111.6,
|
|
"nccl-bw/allreduce_8388608_busbw:0": 89.51,
|
|
"nccl-bw/allreduce_8388608_algbw:0": 51.15,
|
|
"nccl-bw/allreduce_8388608_time:0": 164,
|
|
"nccl-bw/allreduce_16777216_busbw:0": 114.38,
|
|
"nccl-bw/allreduce_16777216_algbw:0": 65.36,
|
|
"nccl-bw/allreduce_16777216_time:0": 256.7,
|
|
"nccl-bw/allreduce_33554432_busbw:0": 154.89,
|
|
"nccl-bw/allreduce_33554432_algbw:0": 88.51,
|
|
"nccl-bw/allreduce_33554432_time:0": 379.1,
|
|
"nccl-bw/allreduce_67108864_busbw:0": 200.01,
|
|
"nccl-bw/allreduce_67108864_algbw:0": 114.29,
|
|
"nccl-bw/allreduce_67108864_time:0": 587.2,
|
|
"nccl-bw/allreduce_134217728_busbw:0": 202.97,
|
|
"nccl-bw/allreduce_134217728_algbw:0": 115.98,
|
|
"nccl-bw/allreduce_134217728_time:0": 1157.2,
|
|
"nccl-bw/allreduce_268435456_busbw:0": 221.82,
|
|
"nccl-bw/allreduce_268435456_algbw:0": 126.75,
|
|
"nccl-bw/allreduce_268435456_time:0": 2117.8,
|
|
"nccl-bw/allreduce_536870912_busbw:0": 224.54,
|
|
"nccl-bw/allreduce_536870912_algbw:0": 128.31,
|
|
"nccl-bw/allreduce_536870912_time:0": 4184.2,
|
|
"nccl-bw/allreduce_1073741824_busbw:0": 230.15,
|
|
"nccl-bw/allreduce_1073741824_algbw:0": 131.51,
|
|
"nccl-bw/allreduce_1073741824_time:0": 8164.5,
|
|
"nccl-bw/allreduce_2147483648_busbw:0": 231.89,
|
|
"nccl-bw/allreduce_2147483648_algbw:0": 132.51,
|
|
"nccl-bw/allreduce_2147483648_time:0": 16207,
|
|
"nccl-bw/allreduce_4294967296_busbw:0": 234.45,
|
|
"nccl-bw/allreduce_4294967296_algbw:0": 133.97,
|
|
"nccl-bw/allreduce_4294967296_time:0": 32059,
|
|
"nccl-bw/allreduce_8589934592_busbw:0": 235.36,
|
|
"nccl-bw/allreduce_8589934592_algbw:0": 134.49,
|
|
"nccl-bw/allreduce_8589934592_time:0": 63870,
|
|
"resnet_models/pytorch-resnet50/steptime_train_float32": 253.9552273229,
|
|
"resnet_models/pytorch-resnet50/throughput_train_float32": 760.334809913,
|
|
"resnet_models/pytorch-resnet50/steptime_train_float16": 200.0860618427,
|
|
"resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430923,
|
|
"resnet_models/pytorch-resnet101/steptime_train_float32": 389.0860509127,
|
|
"resnet_models/pytorch-resnet101/throughput_train_float32": 496.117474093,
|
|
"resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918,
|
|
"resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195,
|
|
"resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278315,
|
|
"resnet_models/pytorch-resnet152/throughput_train_float32": 352.0709954335,
|
|
"resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802,
|
|
"resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998154,
|
|
"pytorch-sharding-matmul/allreduce": 10.574411869,
|
|
"pytorch-sharding-matmul/allgather": 10.0846967697,
|
|
"vgg_models/pytorch-vgg11/steptime_train_float32": 40.3528367169,
|
|
"vgg_models/pytorch-vgg11/throughput_train_float32": 796.361593695,
|
|
"vgg_models/pytorch-vgg11/steptime_train_float16": 24.1335148457,
|
|
"vgg_models/pytorch-vgg11/throughput_train_float16": 1330.4113614585,
|
|
"vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936,
|
|
"vgg_models/pytorch-vgg13/throughput_train_float32": 580.2341074444,
|
|
"vgg_models/pytorch-vgg13/steptime_train_float16": 33.3522899309,
|
|
"vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023902,
|
|
"vgg_models/pytorch-vgg16/steptime_train_float32": 65.225199447,
|
|
"vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638876,
|
|
"vgg_models/pytorch-vgg16/steptime_train_float16": 39.2528773518,
|
|
"vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546148,
|
|
"vgg_models/pytorch-vgg19/steptime_train_float32": 74.9348710524,
|
|
"vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
|
|
"vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
|
|
"vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377
|
|
},
|
|
{
|
|
"index": "sb-validation-03",
|
|
"diagnosis/accept": false,
|
|
"diagnosis/issue_num": 17,
|
|
"diagnosis/category": "FailedTest",
|
|
"diagnosis/issue_details": "mem-bw/D2H_Mem_BW:0_miss,mem-bw/D2H_Mem_BW:1_miss,mem-bw/D2H_Mem_BW:2_miss,mem-bw/D2H_Mem_BW:3_miss,mem-bw/D2H_Mem_BW:4_miss,mem-bw/D2H_Mem_BW:5_miss,mem-bw/D2H_Mem_BW:6_miss,mem-bw/D2H_Mem_BW:7_miss,mem-bw/H2D_Mem_BW:0_miss,mem-bw/H2D_Mem_BW:1_miss,mem-bw/H2D_Mem_BW:2_miss,mem-bw/H2D_Mem_BW:3_miss,mem-bw/H2D_Mem_BW:4_miss,mem-bw/H2D_Mem_BW:5_miss,mem-bw/H2D_Mem_BW:6_miss,mem-bw/H2D_Mem_BW:7_miss,mem-bw/return_code(VAL: 1.0000 Rule:lambda x:x>0)",
|
|
"bert_models/pytorch-bert-base/steptime_train_float32": 114.5916701062,
|
|
"bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591,
|
|
"bert_models/pytorch-bert-base/steptime_train_float16": 83.8895108318,
|
|
"bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582742,
|
|
"bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371914,
|
|
"bert_models/pytorch-bert-large/throughput_train_float32": 103.9487609742,
|
|
"bert_models/pytorch-bert-large/steptime_train_float16": 206.8114168942,
|
|
"bert_models/pytorch-bert-large/throughput_train_float16": 154.8408911711,
|
|
"pytorch-computation-communication-overlap/mul_cost:0": 44.1822062144,
|
|
"pytorch-computation-communication-overlap/mul_cost:1": 44.1822139389,
|
|
"pytorch-computation-communication-overlap/mul_cost:2": 43.9701470781,
|
|
"pytorch-computation-communication-overlap/mul_cost:3": 43.9701478756,
|
|
"pytorch-computation-communication-overlap/mul_cost:4": 43.9701779317,
|
|
"pytorch-computation-communication-overlap/mul_cost:5": 43.9701571606,
|
|
"pytorch-computation-communication-overlap/mul_cost:6": 43.9701651983,
|
|
"pytorch-computation-communication-overlap/mul_cost:7": 44.1795444785,
|
|
"pytorch-computation-communication-overlap/matmul_cost:0": 137.0477370556,
|
|
"pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344693,
|
|
"pytorch-computation-communication-overlap/matmul_cost:2": 137.047772209,
|
|
"pytorch-computation-communication-overlap/matmul_cost:3": 137.0477969726,
|
|
"pytorch-computation-communication-overlap/matmul_cost:4": 137.0481367431,
|
|
"pytorch-computation-communication-overlap/matmul_cost:5": 137.0482198877,
|
|
"pytorch-computation-communication-overlap/matmul_cost:6": 137.0477532237,
|
|
"pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081607,
|
|
"densenet_models/pytorch-densenet169/steptime_train_float32": 150.6415554322,
|
|
"densenet_models/pytorch-densenet169/throughput_train_float32": 212.4701719243,
|
|
"densenet_models/pytorch-densenet169/steptime_train_float16": 145.3081957297,
|
|
"densenet_models/pytorch-densenet169/throughput_train_float16": 220.2932337305,
|
|
"densenet_models/pytorch-densenet201/steptime_train_float32": 182.9111778643,
|
|
"densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232,
|
|
"densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596,
|
|
"densenet_models/pytorch-densenet201/throughput_train_float16": 181.5446573603,
|
|
"gemm-flops/FP64:0": 9031.23,
|
|
"gemm-flops/FP64:1": 9040.85,
|
|
"gemm-flops/FP64:2": 9010.56,
|
|
"gemm-flops/FP64:3": 9041.26,
|
|
"gemm-flops/FP64:4": 9039.19,
|
|
"gemm-flops/FP64:5": 9015.69,
|
|
"gemm-flops/FP64:6": 9022.19,
|
|
"gemm-flops/FP64:7": 9030.2,
|
|
"gemm-flops/FP32:0": 18362.1,
|
|
"gemm-flops/FP32:1": 18375.6,
|
|
"gemm-flops/FP32:2": 18314.9,
|
|
"gemm-flops/FP32:3": 18375.6,
|
|
"gemm-flops/FP32:4": 18368.6,
|
|
"gemm-flops/FP32:5": 18347.1,
|
|
"gemm-flops/FP32:6": 18247.4,
|
|
"gemm-flops/FP32:7": 18318.4,
|
|
"gemm-flops/FP16:0": 33878,
|
|
"gemm-flops/FP16:1": 33911.1,
|
|
"gemm-flops/FP16:2": 33769.3,
|
|
"gemm-flops/FP16:3": 33909.9,
|
|
"gemm-flops/FP16:4": 33896.5,
|
|
"gemm-flops/FP16:5": 33798.1,
|
|
"gemm-flops/FP16:6": 33647.3,
|
|
"gemm-flops/FP16:7": 33764.8,
|
|
"gemm-flops/FP64_TC:0": 18963.6,
|
|
"gemm-flops/FP64_TC:1": 18924.2,
|
|
"gemm-flops/FP64_TC:2": 18930.3,
|
|
"gemm-flops/FP64_TC:3": 18971.9,
|
|
"gemm-flops/FP64_TC:4": 18946,
|
|
"gemm-flops/FP64_TC:5": 18945,
|
|
"gemm-flops/FP64_TC:6": 18822.9,
|
|
"gemm-flops/FP64_TC:7": 18911.1,
|
|
"gemm-flops/TF32_TC:0": 127900,
|
|
"gemm-flops/TF32_TC:1": 129094,
|
|
"gemm-flops/TF32_TC:2": 127831,
|
|
"gemm-flops/TF32_TC:3": 128709,
|
|
"gemm-flops/TF32_TC:4": 127388,
|
|
"gemm-flops/TF32_TC:5": 127861,
|
|
"gemm-flops/TF32_TC:6": 128492,
|
|
"gemm-flops/TF32_TC:7": 127720,
|
|
"gemm-flops/BF16_TC:0": 264965,
|
|
"gemm-flops/BF16_TC:1": 266638,
|
|
"gemm-flops/BF16_TC:2": 263151,
|
|
"gemm-flops/BF16_TC:3": 264752,
|
|
"gemm-flops/BF16_TC:4": 263049,
|
|
"gemm-flops/BF16_TC:5": 266605,
|
|
"gemm-flops/BF16_TC:6": 267501,
|
|
"gemm-flops/BF16_TC:7": 263880,
|
|
"gemm-flops/FP16_TC:0": 279474,
|
|
"gemm-flops/FP16_TC:1": 281256,
|
|
"gemm-flops/FP16_TC:2": 277403,
|
|
"gemm-flops/FP16_TC:3": 279147,
|
|
"gemm-flops/FP16_TC:4": 277587,
|
|
"gemm-flops/FP16_TC:5": 281537,
|
|
"gemm-flops/FP16_TC:6": 282132,
|
|
"gemm-flops/FP16_TC:7": 277788,
|
|
"gemm-flops/INT8_TC:0": 475160,
|
|
"gemm-flops/INT8_TC:1": 477725,
|
|
"gemm-flops/INT8_TC:2": 471621,
|
|
"gemm-flops/INT8_TC:3": 473716,
|
|
"gemm-flops/INT8_TC:4": 472124,
|
|
"gemm-flops/INT8_TC:5": 479972,
|
|
"gemm-flops/INT8_TC:6": 481327,
|
|
"gemm-flops/INT8_TC:7": 474710,
|
|
"gemm-flops/INT4_TC:0": 970330,
|
|
"gemm-flops/INT4_TC:1": 976837,
|
|
"gemm-flops/INT4_TC:2": 966003,
|
|
"gemm-flops/INT4_TC:3": 971315,
|
|
"gemm-flops/INT4_TC:4": 964441,
|
|
"gemm-flops/INT4_TC:5": 982461,
|
|
"gemm-flops/INT4_TC:6": 979610,
|
|
"gemm-flops/INT4_TC:7": 968359,
|
|
"gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971836,
|
|
"gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.1154543969,
|
|
"gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742235,
|
|
"gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.1394499411,
|
|
"gpu-sm-copy-bw/dtoh:0": 3.91755,
|
|
"gpu-sm-copy-bw/dtoh:1": 4.45414,
|
|
"gpu-sm-copy-bw/dtoh:2": 1.26483,
|
|
"gpu-sm-copy-bw/dtoh:3": 1.30041,
|
|
"gpu-sm-copy-bw/dtoh:4": 1.31577,
|
|
"gpu-sm-copy-bw/dtoh:5": 1.27968,
|
|
"gpu-sm-copy-bw/dtoh:6": 4.47849,
|
|
"gpu-sm-copy-bw/dtoh:7": 3.96231,
|
|
"gpu-sm-copy-bw/dtoh:8": 3.91705,
|
|
"gpu-sm-copy-bw/dtoh:9": 4.45487,
|
|
"gpu-sm-copy-bw/dtoh:10": 1.26352,
|
|
"gpu-sm-copy-bw/dtoh:11": 1.2999,
|
|
"gpu-sm-copy-bw/dtoh:12": 1.31677,
|
|
"gpu-sm-copy-bw/dtoh:13": 1.27885,
|
|
"gpu-sm-copy-bw/dtoh:14": 4.47913,
|
|
"gpu-sm-copy-bw/dtoh:15": 3.95893,
|
|
"gpu-sm-copy-bw/dtoh:16": 3.91729,
|
|
"gpu-sm-copy-bw/dtoh:17": 4.45627,
|
|
"gpu-sm-copy-bw/dtoh:18": 1.26437,
|
|
"gpu-sm-copy-bw/dtoh:19": 1.30144,
|
|
"gpu-sm-copy-bw/dtoh:20": 1.31704,
|
|
"gpu-sm-copy-bw/dtoh:21": 1.27857,
|
|
"gpu-sm-copy-bw/dtoh:22": 4.47889,
|
|
"gpu-sm-copy-bw/dtoh:23": 3.95984,
|
|
"gpu-sm-copy-bw/dtoh:24": 3.92025,
|
|
"gpu-sm-copy-bw/dtoh:25": 4.45423,
|
|
"gpu-sm-copy-bw/dtoh:26": 1.26449,
|
|
"gpu-sm-copy-bw/dtoh:27": 1.29954,
|
|
"gpu-sm-copy-bw/dtoh:28": 1.31731,
|
|
"gpu-sm-copy-bw/dtoh:29": 1.27916,
|
|
"gpu-sm-copy-bw/dtoh:30": 4.4797,
|
|
"gpu-sm-copy-bw/dtoh:31": 3.96124,
|
|
"gpu-sm-copy-bw/htod:0": 23.9685,
|
|
"gpu-sm-copy-bw/htod:1": 23.967,
|
|
"gpu-sm-copy-bw/htod:2": 19.9898,
|
|
"gpu-sm-copy-bw/htod:3": 20.2848,
|
|
"gpu-sm-copy-bw/htod:4": 20.3931,
|
|
"gpu-sm-copy-bw/htod:5": 20.0888,
|
|
"gpu-sm-copy-bw/htod:6": 23.9766,
|
|
"gpu-sm-copy-bw/htod:7": 23.9792,
|
|
"gpu-sm-copy-bw/htod:8": 23.9497,
|
|
"gpu-sm-copy-bw/htod:9": 23.9438,
|
|
"gpu-sm-copy-bw/htod:10": 20.0039,
|
|
"gpu-sm-copy-bw/htod:11": 20.2469,
|
|
"gpu-sm-copy-bw/htod:12": 20.4344,
|
|
"gpu-sm-copy-bw/htod:13": 20.1005,
|
|
"gpu-sm-copy-bw/htod:14": 23.9491,
|
|
"gpu-sm-copy-bw/htod:15": 23.9898,
|
|
"gpu-sm-copy-bw/htod:16": 23.962,
|
|
"gpu-sm-copy-bw/htod:17": 23.97,
|
|
"gpu-sm-copy-bw/htod:18": 19.987,
|
|
"gpu-sm-copy-bw/htod:19": 20.1549,
|
|
"gpu-sm-copy-bw/htod:20": 20.3931,
|
|
"gpu-sm-copy-bw/htod:21": 20.1013,
|
|
"gpu-sm-copy-bw/htod:22": 23.9865,
|
|
"gpu-sm-copy-bw/htod:23": 23.972,
|
|
"gpu-sm-copy-bw/htod:24": 23.9569,
|
|
"gpu-sm-copy-bw/htod:25": 23.9762,
|
|
"gpu-sm-copy-bw/htod:26": 19.92,
|
|
"gpu-sm-copy-bw/htod:27": 20.2638,
|
|
"gpu-sm-copy-bw/htod:28": 20.4419,
|
|
"gpu-sm-copy-bw/htod:29": 20.1054,
|
|
"gpu-sm-copy-bw/htod:30": 23.9752,
|
|
"gpu-sm-copy-bw/htod:31": 23.9631,
|
|
"ib-loopback/IB_write_512_Avg_0:0": 1492.89,
|
|
"ib-loopback/IB_write_1024_Avg_0:0": 3224.92,
|
|
"ib-loopback/IB_write_2048_Avg_0:0": 6714.5,
|
|
"ib-loopback/IB_write_4096_Avg_0:0": 12871.93,
|
|
"ib-loopback/IB_write_8192_Avg_0:0": 19990.78,
|
|
"ib-loopback/IB_write_16384_Avg_0:0": 22172.25,
|
|
"ib-loopback/IB_write_32768_Avg_0:0": 23073.19,
|
|
"ib-loopback/IB_write_65536_Avg_0:0": 23527.09,
|
|
"ib-loopback/IB_write_131072_Avg_0:0": 23805.92,
|
|
"ib-loopback/IB_write_262144_Avg_0:0": 23380.51,
|
|
"ib-loopback/IB_write_524288_Avg_0:0": 23856.63,
|
|
"ib-loopback/IB_write_1048576_Avg_0:0": 23869.58,
|
|
"ib-loopback/IB_write_2097152_Avg_0:0": 23885.7,
|
|
"ib-loopback/IB_write_4194304_Avg_0:0": 23914.56,
|
|
"ib-loopback/IB_write_8388608_Avg_0:0": 23935.21,
|
|
"ib-loopback/IB_write_512_Avg_1:0": 1479.1,
|
|
"ib-loopback/IB_write_1024_Avg_1:0": 3246.48,
|
|
"ib-loopback/IB_write_2048_Avg_1:0": 6754.94,
|
|
"ib-loopback/IB_write_4096_Avg_1:0": 13101.09,
|
|
"ib-loopback/IB_write_8192_Avg_1:0": 19945.81,
|
|
"ib-loopback/IB_write_16384_Avg_1:0": 22631.62,
|
|
"ib-loopback/IB_write_32768_Avg_1:0": 23381.58,
|
|
"ib-loopback/IB_write_65536_Avg_1:0": 23515.29,
|
|
"ib-loopback/IB_write_131072_Avg_1:0": 23756.01,
|
|
"ib-loopback/IB_write_262144_Avg_1:0": 23875.91,
|
|
"ib-loopback/IB_write_524288_Avg_1:0": 23911.13,
|
|
"ib-loopback/IB_write_1048576_Avg_1:0": 23935.42,
|
|
"ib-loopback/IB_write_2097152_Avg_1:0": 23941.25,
|
|
"ib-loopback/IB_write_4194304_Avg_1:0": 23922.06,
|
|
"ib-loopback/IB_write_8388608_Avg_1:0": 23928.59,
|
|
"ib-loopback/IB_write_512_Avg_2:0": 1505.29,
|
|
"ib-loopback/IB_write_1024_Avg_2:0": 3215.97,
|
|
"ib-loopback/IB_write_2048_Avg_2:0": 6745.49,
|
|
"ib-loopback/IB_write_4096_Avg_2:0": 11548.88,
|
|
"ib-loopback/IB_write_8192_Avg_2:0": 19432.15,
|
|
"ib-loopback/IB_write_16384_Avg_2:0": 22765.51,
|
|
"ib-loopback/IB_write_32768_Avg_2:0": 23235.07,
|
|
"ib-loopback/IB_write_65536_Avg_2:0": 23620.08,
|
|
"ib-loopback/IB_write_131072_Avg_2:0": 23759.08,
|
|
"ib-loopback/IB_write_262144_Avg_2:0": 23859.82,
|
|
"ib-loopback/IB_write_524288_Avg_2:0": 23775.01,
|
|
"ib-loopback/IB_write_1048576_Avg_2:0": 23885.96,
|
|
"ib-loopback/IB_write_2097152_Avg_2:0": 23894.73,
|
|
"ib-loopback/IB_write_4194304_Avg_2:0": 23909.98,
|
|
"ib-loopback/IB_write_8388608_Avg_2:0": 23927.21,
|
|
"ib-loopback/IB_write_512_Avg_3:0": 1474.47,
|
|
"ib-loopback/IB_write_1024_Avg_3:0": 3344.68,
|
|
"ib-loopback/IB_write_2048_Avg_3:0": 6606.35,
|
|
"ib-loopback/IB_write_4096_Avg_3:0": 12071.4,
|
|
"ib-loopback/IB_write_8192_Avg_3:0": 18408.33,
|
|
"ib-loopback/IB_write_16384_Avg_3:0": 20789.27,
|
|
"ib-loopback/IB_write_32768_Avg_3:0": 22469.43,
|
|
"ib-loopback/IB_write_65536_Avg_3:0": 22777.1,
|
|
"ib-loopback/IB_write_131072_Avg_3:0": 23461.03,
|
|
"ib-loopback/IB_write_262144_Avg_3:0": 23397.19,
|
|
"ib-loopback/IB_write_524288_Avg_3:0": 23526.55,
|
|
"ib-loopback/IB_write_1048576_Avg_3:0": 23854.76,
|
|
"ib-loopback/IB_write_2097152_Avg_3:0": 23862.23,
|
|
"ib-loopback/IB_write_4194304_Avg_3:0": 23931.15,
|
|
"ib-loopback/IB_write_8388608_Avg_3:0": 23924.44,
|
|
"ib-loopback/IB_write_512_Avg_4:0": 1523.33,
|
|
"ib-loopback/IB_write_1024_Avg_4:0": 3233.23,
|
|
"ib-loopback/IB_write_2048_Avg_4:0": 6792.88,
|
|
"ib-loopback/IB_write_4096_Avg_4:0": 12616.05,
|
|
"ib-loopback/IB_write_8192_Avg_4:0": 19324.05,
|
|
"ib-loopback/IB_write_16384_Avg_4:0": 22082.51,
|
|
"ib-loopback/IB_write_32768_Avg_4:0": 23294.23,
|
|
"ib-loopback/IB_write_65536_Avg_4:0": 23546.22,
|
|
"ib-loopback/IB_write_131072_Avg_4:0": 23727.91,
|
|
"ib-loopback/IB_write_262144_Avg_4:0": 23843.93,
|
|
"ib-loopback/IB_write_524288_Avg_4:0": 23905.96,
|
|
"ib-loopback/IB_write_1048576_Avg_4:0": 23902.37,
|
|
"ib-loopback/IB_write_2097152_Avg_4:0": 23921.03,
|
|
"ib-loopback/IB_write_4194304_Avg_4:0": 23921.06,
|
|
"ib-loopback/IB_write_8388608_Avg_4:0": 23922.4,
|
|
"ib-loopback/IB_write_512_Avg_5:0": 1506.63,
|
|
"ib-loopback/IB_write_1024_Avg_5:0": 3261.71,
|
|
"ib-loopback/IB_write_2048_Avg_5:0": 6752.38,
|
|
"ib-loopback/IB_write_4096_Avg_5:0": 13592.41,
|
|
"ib-loopback/IB_write_8192_Avg_5:0": 19989.31,
|
|
"ib-loopback/IB_write_16384_Avg_5:0": 22560.57,
|
|
"ib-loopback/IB_write_32768_Avg_5:0": 23389.29,
|
|
"ib-loopback/IB_write_65536_Avg_5:0": 23503.32,
|
|
"ib-loopback/IB_write_131072_Avg_5:0": 23741.51,
|
|
"ib-loopback/IB_write_262144_Avg_5:0": 23866.43,
|
|
"ib-loopback/IB_write_524288_Avg_5:0": 23898.95,
|
|
"ib-loopback/IB_write_1048576_Avg_5:0": 23876.36,
|
|
"ib-loopback/IB_write_2097152_Avg_5:0": 23919.63,
|
|
"ib-loopback/IB_write_4194304_Avg_5:0": 23924.68,
|
|
"ib-loopback/IB_write_8388608_Avg_5:0": 23930.37,
|
|
"ib-loopback/IB_write_512_Avg_6:0": 1467.69,
|
|
"ib-loopback/IB_write_1024_Avg_6:0": 3157.04,
|
|
"ib-loopback/IB_write_2048_Avg_6:0": 6494.61,
|
|
"ib-loopback/IB_write_4096_Avg_6:0": 12883.51,
|
|
"ib-loopback/IB_write_8192_Avg_6:0": 19207.67,
|
|
"ib-loopback/IB_write_16384_Avg_6:0": 22519.39,
|
|
"ib-loopback/IB_write_32768_Avg_6:0": 23323.46,
|
|
"ib-loopback/IB_write_65536_Avg_6:0": 23523.6,
|
|
"ib-loopback/IB_write_131072_Avg_6:0": 23626.67,
|
|
"ib-loopback/IB_write_262144_Avg_6:0": 23836.99,
|
|
"ib-loopback/IB_write_524288_Avg_6:0": 23904.51,
|
|
"ib-loopback/IB_write_1048576_Avg_6:0": 23919.07,
|
|
"ib-loopback/IB_write_2097152_Avg_6:0": 23943.82,
|
|
"ib-loopback/IB_write_4194304_Avg_6:0": 23936.77,
|
|
"ib-loopback/IB_write_8388608_Avg_6:0": 23941.57,
|
|
"ib-loopback/IB_write_512_Avg_7:0": 1505.63,
|
|
"ib-loopback/IB_write_1024_Avg_7:0": 3259.93,
|
|
"ib-loopback/IB_write_2048_Avg_7:0": 6738.6,
|
|
"ib-loopback/IB_write_4096_Avg_7:0": 13352.06,
|
|
"ib-loopback/IB_write_8192_Avg_7:0": 19941.35,
|
|
"ib-loopback/IB_write_16384_Avg_7:0": 22566.09,
|
|
"ib-loopback/IB_write_32768_Avg_7:0": 23244.77,
|
|
"ib-loopback/IB_write_65536_Avg_7:0": 23377.67,
|
|
"ib-loopback/IB_write_131072_Avg_7:0": 23736.17,
|
|
"ib-loopback/IB_write_262144_Avg_7:0": 23829.25,
|
|
"ib-loopback/IB_write_524288_Avg_7:0": 23879.6,
|
|
"ib-loopback/IB_write_1048576_Avg_7:0": 23895.1,
|
|
"ib-loopback/IB_write_2097152_Avg_7:0": 23930.64,
|
|
"ib-loopback/IB_write_4194304_Avg_7:0": 23845.63,
|
|
"ib-loopback/IB_write_8388608_Avg_7:0": 23896.94,
|
|
"kernel-launch/return_code": 0,
|
|
"kernel-launch/event_overhead:0": 0.00596,
|
|
"kernel-launch/event_overhead:1": 0.00595,
|
|
"kernel-launch/event_overhead:2": 0.00557,
|
|
"kernel-launch/event_overhead:3": 0.0055,
|
|
"kernel-launch/event_overhead:4": 0.00592,
|
|
"kernel-launch/event_overhead:5": 0.00589,
|
|
"kernel-launch/event_overhead:6": 0.00572,
|
|
"kernel-launch/event_overhead:7": 0.0059,
|
|
"kernel-launch/wall_overhead:0": 0.01026,
|
|
"kernel-launch/wall_overhead:1": 0.01026,
|
|
"kernel-launch/wall_overhead:2": 0.01046,
|
|
"kernel-launch/wall_overhead:3": 0.01049,
|
|
"kernel-launch/wall_overhead:4": 0.01063,
|
|
"kernel-launch/wall_overhead:5": 0.01006,
|
|
"kernel-launch/wall_overhead:6": 0.01045,
|
|
"kernel-launch/wall_overhead:7": 0.01071,
|
|
"lstm_models/pytorch-lstm/steptime_train_float32": 48.0702451896,
|
|
"lstm_models/pytorch-lstm/throughput_train_float32": 4806.4724411328,
|
|
"lstm_models/pytorch-lstm/steptime_train_float16": 25.9531298652,
|
|
"lstm_models/pytorch-lstm/throughput_train_float16": 9069.9080925588,
|
|
"pytorch-matmul/nosharding": 34.6449975967,
|
|
"mem-bw/return_code": 1,
|
|
"mem-bw/H2D_Mem_BW:0": "N/A",
|
|
"mem-bw/H2D_Mem_BW:1": "N/A",
|
|
"mem-bw/H2D_Mem_BW:2": "N/A",
|
|
"mem-bw/H2D_Mem_BW:3": "N/A",
|
|
"mem-bw/H2D_Mem_BW:4": "N/A",
|
|
"mem-bw/H2D_Mem_BW:5": "N/A",
|
|
"mem-bw/H2D_Mem_BW:6": "N/A",
|
|
"mem-bw/H2D_Mem_BW:7": "N/A",
|
|
"mem-bw/D2H_Mem_BW:0": "N/A",
|
|
"mem-bw/D2H_Mem_BW:1": "N/A",
|
|
"mem-bw/D2H_Mem_BW:2": "N/A",
|
|
"mem-bw/D2H_Mem_BW:3": "N/A",
|
|
"mem-bw/D2H_Mem_BW:4": "N/A",
|
|
"mem-bw/D2H_Mem_BW:5": "N/A",
|
|
"mem-bw/D2H_Mem_BW:6": "N/A",
|
|
"mem-bw/D2H_Mem_BW:7": "N/A",
|
|
"mem-bw/D2D_Mem_BW:0": "N/A",
|
|
"mem-bw/D2D_Mem_BW:1": "N/A",
|
|
"mem-bw/D2D_Mem_BW:2": "N/A",
|
|
"mem-bw/D2D_Mem_BW:3": "N/A",
|
|
"mem-bw/D2D_Mem_BW:4": "N/A",
|
|
"mem-bw/D2D_Mem_BW:5": "N/A",
|
|
"mem-bw/D2D_Mem_BW:6": "N/A",
|
|
"mem-bw/D2D_Mem_BW:7": "N/A",
|
|
"nccl-bw/allreduce_8_busbw:0": 0,
|
|
"nccl-bw/allreduce_8_algbw:0": 0,
|
|
"nccl-bw/allreduce_8_time:0": 37.84,
|
|
"nccl-bw/allreduce_16_busbw:0": 0,
|
|
"nccl-bw/allreduce_16_algbw:0": 0,
|
|
"nccl-bw/allreduce_16_time:0": 36.42,
|
|
"nccl-bw/allreduce_32_busbw:0": 0,
|
|
"nccl-bw/allreduce_32_algbw:0": 0,
|
|
"nccl-bw/allreduce_32_time:0": 36.87,
|
|
"nccl-bw/allreduce_64_busbw:0": 0,
|
|
"nccl-bw/allreduce_64_algbw:0": 0,
|
|
"nccl-bw/allreduce_64_time:0": 35.83,
|
|
"nccl-bw/allreduce_128_busbw:0": 0.01,
|
|
"nccl-bw/allreduce_128_algbw:0": 0,
|
|
"nccl-bw/allreduce_128_time:0": 36.91,
|
|
"nccl-bw/allreduce_256_busbw:0": 0.01,
|
|
"nccl-bw/allreduce_256_algbw:0": 0.01,
|
|
"nccl-bw/allreduce_256_time:0": 37.58,
|
|
"nccl-bw/allreduce_512_busbw:0": 0.02,
|
|
"nccl-bw/allreduce_512_algbw:0": 0.01,
|
|
"nccl-bw/allreduce_512_time:0": 36.98,
|
|
"nccl-bw/allreduce_1024_busbw:0": 0.05,
|
|
"nccl-bw/allreduce_1024_algbw:0": 0.03,
|
|
"nccl-bw/allreduce_1024_time:0": 36.93,
|
|
"nccl-bw/allreduce_2048_busbw:0": 0.1,
|
|
"nccl-bw/allreduce_2048_algbw:0": 0.06,
|
|
"nccl-bw/allreduce_2048_time:0": 36.06,
|
|
"nccl-bw/allreduce_4096_busbw:0": 0.19,
|
|
"nccl-bw/allreduce_4096_algbw:0": 0.11,
|
|
"nccl-bw/allreduce_4096_time:0": 37.2,
|
|
"nccl-bw/allreduce_8192_busbw:0": 0.39,
|
|
"nccl-bw/allreduce_8192_algbw:0": 0.22,
|
|
"nccl-bw/allreduce_8192_time:0": 37.04,
|
|
"nccl-bw/allreduce_16384_busbw:0": 0.77,
|
|
"nccl-bw/allreduce_16384_algbw:0": 0.44,
|
|
"nccl-bw/allreduce_16384_time:0": 37.46,
|
|
"nccl-bw/allreduce_32768_busbw:0": 1.52,
|
|
"nccl-bw/allreduce_32768_algbw:0": 0.87,
|
|
"nccl-bw/allreduce_32768_time:0": 37.64,
|
|
"nccl-bw/allreduce_65536_busbw:0": 3,
|
|
"nccl-bw/allreduce_65536_algbw:0": 1.71,
|
|
"nccl-bw/allreduce_65536_time:0": 38.22,
|
|
"nccl-bw/allreduce_131072_busbw:0": 5.31,
|
|
"nccl-bw/allreduce_131072_algbw:0": 3.04,
|
|
"nccl-bw/allreduce_131072_time:0": 43.17,
|
|
"nccl-bw/allreduce_262144_busbw:0": 9.5,
|
|
"nccl-bw/allreduce_262144_algbw:0": 5.43,
|
|
"nccl-bw/allreduce_262144_time:0": 48.29,
|
|
"nccl-bw/allreduce_524288_busbw:0": 15.11,
|
|
"nccl-bw/allreduce_524288_algbw:0": 8.64,
|
|
"nccl-bw/allreduce_524288_time:0": 60.71,
|
|
"nccl-bw/allreduce_1048576_busbw:0": 24.1,
|
|
"nccl-bw/allreduce_1048576_algbw:0": 13.77,
|
|
"nccl-bw/allreduce_1048576_time:0": 76.13,
|
|
"nccl-bw/allreduce_2097152_busbw:0": 38.12,
|
|
"nccl-bw/allreduce_2097152_algbw:0": 21.78,
|
|
"nccl-bw/allreduce_2097152_time:0": 96.28,
|
|
"nccl-bw/allreduce_4194304_busbw:0": 65.75,
|
|
"nccl-bw/allreduce_4194304_algbw:0": 37.57,
|
|
"nccl-bw/allreduce_4194304_time:0": 111.6,
|
|
"nccl-bw/allreduce_8388608_busbw:0": 89.51,
|
|
"nccl-bw/allreduce_8388608_algbw:0": 51.15,
|
|
"nccl-bw/allreduce_8388608_time:0": 164,
|
|
"nccl-bw/allreduce_16777216_busbw:0": 114.38,
|
|
"nccl-bw/allreduce_16777216_algbw:0": 65.36,
|
|
"nccl-bw/allreduce_16777216_time:0": 256.7,
|
|
"nccl-bw/allreduce_33554432_busbw:0": 154.89,
|
|
"nccl-bw/allreduce_33554432_algbw:0": 88.51,
|
|
"nccl-bw/allreduce_33554432_time:0": 379.1,
|
|
"nccl-bw/allreduce_67108864_busbw:0": 200.01,
|
|
"nccl-bw/allreduce_67108864_algbw:0": 114.29,
|
|
"nccl-bw/allreduce_67108864_time:0": 587.2,
|
|
"nccl-bw/allreduce_134217728_busbw:0": 202.97,
|
|
"nccl-bw/allreduce_134217728_algbw:0": 115.98,
|
|
"nccl-bw/allreduce_134217728_time:0": 1157.2,
|
|
"nccl-bw/allreduce_268435456_busbw:0": 221.82,
|
|
"nccl-bw/allreduce_268435456_algbw:0": 126.75,
|
|
"nccl-bw/allreduce_268435456_time:0": 2117.8,
|
|
"nccl-bw/allreduce_536870912_busbw:0": 224.54,
|
|
"nccl-bw/allreduce_536870912_algbw:0": 128.31,
|
|
"nccl-bw/allreduce_536870912_time:0": 4184.2,
|
|
"nccl-bw/allreduce_1073741824_busbw:0": 230.15,
|
|
"nccl-bw/allreduce_1073741824_algbw:0": 131.51,
|
|
"nccl-bw/allreduce_1073741824_time:0": 8164.5,
|
|
"nccl-bw/allreduce_2147483648_busbw:0": 231.89,
|
|
"nccl-bw/allreduce_2147483648_algbw:0": 132.51,
|
|
"nccl-bw/allreduce_2147483648_time:0": 16207,
|
|
"nccl-bw/allreduce_4294967296_busbw:0": 234.45,
|
|
"nccl-bw/allreduce_4294967296_algbw:0": 133.97,
|
|
"nccl-bw/allreduce_4294967296_time:0": 32059,
|
|
"nccl-bw/allreduce_8589934592_busbw:0": 235.36,
|
|
"nccl-bw/allreduce_8589934592_algbw:0": 134.49,
|
|
"nccl-bw/allreduce_8589934592_time:0": 63870,
|
|
"resnet_models/pytorch-resnet50/steptime_train_float32": 253.9552273229,
|
|
"resnet_models/pytorch-resnet50/throughput_train_float32": 760.334809913,
|
|
"resnet_models/pytorch-resnet50/steptime_train_float16": 200.0860618427,
|
|
"resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430923,
|
|
"resnet_models/pytorch-resnet101/steptime_train_float32": 389.0860509127,
|
|
"resnet_models/pytorch-resnet101/throughput_train_float32": 496.117474093,
|
|
"resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918,
|
|
"resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195,
|
|
"resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278315,
|
|
"resnet_models/pytorch-resnet152/throughput_train_float32": 352.0709954335,
|
|
"resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802,
|
|
"resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998154,
|
|
"pytorch-sharding-matmul/allreduce": 10.574411869,
|
|
"pytorch-sharding-matmul/allgather": 10.0846967697,
|
|
"vgg_models/pytorch-vgg11/steptime_train_float32": 40.3528367169,
|
|
"vgg_models/pytorch-vgg11/throughput_train_float32": 796.361593695,
|
|
"vgg_models/pytorch-vgg11/steptime_train_float16": 24.1335148457,
|
|
"vgg_models/pytorch-vgg11/throughput_train_float16": 1330.4113614585,
|
|
"vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936,
|
|
"vgg_models/pytorch-vgg13/throughput_train_float32": 580.2341074444,
|
|
"vgg_models/pytorch-vgg13/steptime_train_float16": 33.3522899309,
|
|
"vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023902,
|
|
"vgg_models/pytorch-vgg16/steptime_train_float32": 65.225199447,
|
|
"vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638876,
|
|
"vgg_models/pytorch-vgg16/steptime_train_float16": 39.2528773518,
|
|
"vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546148,
|
|
"vgg_models/pytorch-vgg19/steptime_train_float32": 74.9348710524,
|
|
"vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
|
|
"vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
|
|
"vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377
|
|
}
|
|
] |