From f841c8f466cc3df7e15756441c1ca4872af9a08c Mon Sep 17 00:00:00 2001 From: guoshzhao Date: Fri, 22 Oct 2021 15:22:15 +0800 Subject: [PATCH] Benchmarks: Add Feature - Support AMD and CUDA platform for DockerBenchmark. (#226) Description Add CudaDockerBenchmark and RocmDockerBenchmark to support amd and cuda platform for DockerBenchmark. --- .azure-pipelines/cuda-unit-test.yml | 1 + .../benchmarks/docker_benchmarks/__init__.py | 5 +- .../docker_benchmarks/docker_base.py | 51 ++++++++++++ .../docker_benchmarks/test_docker_base.py | 82 +++++++++++++++++-- 4 files changed, 129 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/cuda-unit-test.yml b/.azure-pipelines/cuda-unit-test.yml index dd988545..4c747f0d 100644 --- a/.azure-pipelines/cuda-unit-test.yml +++ b/.azure-pipelines/cuda-unit-test.yml @@ -11,6 +11,7 @@ pool: container: image: nvcr.io/nvidia/pytorch:20.12-py3 + options: '-v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker' steps: - script: | diff --git a/superbench/benchmarks/docker_benchmarks/__init__.py b/superbench/benchmarks/docker_benchmarks/__init__.py index caac572a..02e57aad 100644 --- a/superbench/benchmarks/docker_benchmarks/__init__.py +++ b/superbench/benchmarks/docker_benchmarks/__init__.py @@ -3,6 +3,7 @@ """A module containing all the benchmarks packaged in docker.""" -from superbench.benchmarks.docker_benchmarks.docker_base import DockerBenchmark +from superbench.benchmarks.docker_benchmarks.docker_base import DockerBenchmark, CudaDockerBenchmark, \ + RocmDockerBenchmark -__all__ = ['DockerBenchmark'] +__all__ = ['DockerBenchmark', 'CudaDockerBenchmark', 'RocmDockerBenchmark'] diff --git a/superbench/benchmarks/docker_benchmarks/docker_base.py b/superbench/benchmarks/docker_benchmarks/docker_base.py index ca833fc9..901889c8 100644 --- a/superbench/benchmarks/docker_benchmarks/docker_base.py +++ b/superbench/benchmarks/docker_benchmarks/docker_base.py @@ -31,6 +31,18 @@ class DockerBenchmark(Benchmark): # Container name of the current docker-benchmark. self._container_name = None + # Default options for docker run. + self._default_options = '-i --rm --privileged --net=host --ipc=host' + + # Platform-specific options for docker run. + self._platform_options = None + + # Entrypoint option of the current docker-benchmark. + self._entrypoint = None + + # CMD option of the current docker-benchmark. + self._cmd = None + ''' # If need to add new arguments, super().add_parser_arguments() must be called. def add_parser_arguments(self): @@ -67,6 +79,19 @@ class DockerBenchmark(Benchmark): ) return False + command = 'docker run ' + command += self._default_options + command += ' --name={container_name} {platform_options} {entrypoint} {image} {cmd}' + self._commands.append( + command.format( + container_name=self._container_name, + platform_options=self._platform_options or '', + entrypoint='' if self._entrypoint is None else '--entrypoint {}'.format(self._entrypoint), + image=self._image_uri, + cmd=self._cmd or '' + ) + ) + return True def _postprocess(self): @@ -132,3 +157,29 @@ class DockerBenchmark(Benchmark): """Print environments or dependencies information.""" # TODO: will implement it when add real benchmarks in the future. pass + + +class CudaDockerBenchmark(DockerBenchmark): + """The base class of benchmarks packaged in nvidia docker container.""" + def __init__(self, name, parameters=''): + """Constructor. + + Args: + name (str): benchmark name. + parameters (str): benchmark parameters. + """ + super().__init__(name, parameters) + self._platform_options = '--gpus=all' + + +class RocmDockerBenchmark(DockerBenchmark): + """The base class of benchmarks packaged in amd docker container.""" + def __init__(self, name, parameters=''): + """Constructor. + + Args: + name (str): benchmark name. + parameters (str): benchmark parameters. + """ + super().__init__(name, parameters) + self._platform_options = '--security-opt seccomp=unconfined --group-add video' diff --git a/tests/benchmarks/docker_benchmarks/test_docker_base.py b/tests/benchmarks/docker_benchmarks/test_docker_base.py index 56a6352c..5d979104 100644 --- a/tests/benchmarks/docker_benchmarks/test_docker_base.py +++ b/tests/benchmarks/docker_benchmarks/test_docker_base.py @@ -5,8 +5,9 @@ import re +from tests.helper import decorator from superbench.benchmarks import BenchmarkType, ReturnCode -from superbench.benchmarks.docker_benchmarks import DockerBenchmark +from superbench.benchmarks.docker_benchmarks import DockerBenchmark, CudaDockerBenchmark, RocmDockerBenchmark class FakeDockerBenchmark(DockerBenchmark): @@ -49,8 +50,61 @@ class FakeDockerBenchmark(DockerBenchmark): return True +class FakeCudaDockerBenchmark(CudaDockerBenchmark): + """Fake benchmark inherit from CudaDockerBenchmark.""" + def __init__(self, name, parameters=''): + """Constructor. + + Args: + name: benchmark name. + parameters: benchmark parameters. + """ + super().__init__(name, parameters) + + def _process_raw_result(self, cmd_idx, raw_output): + """Function to process raw results and save the summarized results. + + self._result.add_raw_data() and self._result.add_result() need to be called to save the results. + + Args: + cmd_idx (int): the index of command corresponding with the raw_output. + raw_output (str): raw output string of the docker-benchmark. + + Return: + True if the raw output string is valid and result can be extracted. + """ + return True + + +class FakeRocmDockerBenchmark(RocmDockerBenchmark): + """Fake benchmark inherit from RocmDockerBenchmark.""" + def __init__(self, name, parameters=''): + """Constructor. + + Args: + name: benchmark name. + parameters: benchmark parameters. + """ + super().__init__(name, parameters) + + def _process_raw_result(self, cmd_idx, raw_output): + """Function to process raw results and save the summarized results. + + self._result.add_raw_data() and self._result.add_result() need to be called to save the results. + + Args: + cmd_idx (int): the index of command corresponding with the raw_output. + raw_output (str): raw output string of the docker-benchmark. + + Return: + True if the raw output string is valid and result can be extracted. + """ + return True + + +@decorator.cuda_test def test_docker_benchmark_base(): - """Test MicroBenchmarkWithInvoke.""" + """Test DockerBenchmark.""" # Negative case - DOCKERBENCHMARK_IMAGE_NOT_SET. benchmark = FakeDockerBenchmark('fake') assert (benchmark._benchmark_type == BenchmarkType.DOCKER) @@ -70,9 +124,21 @@ def test_docker_benchmark_base(): assert (benchmark.run() is False) assert (benchmark.return_code == ReturnCode.DOCKERBENCHMARK_IMAGE_PULL_FAILURE) - # Test for DockerBenchmark._benchmark(). - benchmark._commands.append("echo -n 'cost1: 10.2, cost2: 20.2'") - benchmark._benchmark() - assert (benchmark.raw_data['raw_output_0'] == ['cost1: 10.2, cost2: 20.2']) - assert (benchmark.result['cost1'] == [10.2]) - assert (benchmark.result['cost2'] == [20.2]) + # Positive case + benchmark = FakeDockerBenchmark('fake') + benchmark._image_uri = 'ubuntu' + benchmark._container_name = 'fake-docker-benchmark-test' + benchmark._entrypoint = 'echo' + benchmark._cmd = '-n "cost1: 10.2, cost2: 20.2"' + benchmark.run() + assert ( + benchmark._commands[0] == + 'docker run -i --rm --privileged --net=host --ipc=host --name=fake-docker-benchmark-test' + ' --entrypoint echo ubuntu -n "cost1: 10.2, cost2: 20.2"' + ) + + # Test for _platform_options. + benchmark = FakeCudaDockerBenchmark('fake') + assert (benchmark._platform_options == '--gpus=all') + benchmark = FakeRocmDockerBenchmark('fake') + assert (benchmark._platform_options == '--security-opt seccomp=unconfined --group-add video')