Benchmarks: Add Feature - Support AMD and CUDA platform for DockerBenchmark. (#226)

Description
Add CudaDockerBenchmark and RocmDockerBenchmark to support amd and cuda platform for DockerBenchmark.

.azure-pipelines/cuda-unit-test.yml

@@ -11,6 +11,7 @@ pool:
 
 container:
   image: nvcr.io/nvidia/pytorch:20.12-py3
+  options: '-v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker'
 
 steps:
   - script: |

superbench/benchmarks/docker_benchmarks/__init__.py

@@ -3,6 +3,7 @@
 
 """A module containing all the benchmarks packaged in docker."""
 
-from superbench.benchmarks.docker_benchmarks.docker_base import DockerBenchmark
+from superbench.benchmarks.docker_benchmarks.docker_base import DockerBenchmark, CudaDockerBenchmark, \
+    RocmDockerBenchmark
 
-__all__ = ['DockerBenchmark']
+__all__ = ['DockerBenchmark', 'CudaDockerBenchmark', 'RocmDockerBenchmark']

superbench/benchmarks/docker_benchmarks/docker_base.py

@@ -31,6 +31,18 @@ class DockerBenchmark(Benchmark):
         # Container name of the current docker-benchmark.
         self._container_name = None
 
+        # Default options for docker run.
+        self._default_options = '-i --rm --privileged --net=host --ipc=host'
+
+        # Platform-specific options for docker run.
+        self._platform_options = None
+
+        # Entrypoint option of the current docker-benchmark.
+        self._entrypoint = None
+
+        # CMD option of the current docker-benchmark.
+        self._cmd = None
+
     '''
     # If need to add new arguments, super().add_parser_arguments() must be called.
     def add_parser_arguments(self):
@@ -67,6 +79,19 @@ class DockerBenchmark(Benchmark):
             )
             return False
 
+        command = 'docker run '
+        command += self._default_options
+        command += ' --name={container_name} {platform_options} {entrypoint} {image} {cmd}'
+        self._commands.append(
+            command.format(
+                container_name=self._container_name,
+                platform_options=self._platform_options or '',
+                entrypoint='' if self._entrypoint is None else '--entrypoint {}'.format(self._entrypoint),
+                image=self._image_uri,
+                cmd=self._cmd or ''
+            )
+        )
+
         return True
 
     def _postprocess(self):
@@ -132,3 +157,29 @@ class DockerBenchmark(Benchmark):
         """Print environments or dependencies information."""
         # TODO: will implement it when add real benchmarks in the future.
         pass
+
+
+class CudaDockerBenchmark(DockerBenchmark):
+    """The base class of benchmarks packaged in nvidia docker container."""
+    def __init__(self, name, parameters=''):
+        """Constructor.
+
+        Args:
+            name (str): benchmark name.
+            parameters (str): benchmark parameters.
+        """
+        super().__init__(name, parameters)
+        self._platform_options = '--gpus=all'
+
+
+class RocmDockerBenchmark(DockerBenchmark):
+    """The base class of benchmarks packaged in amd docker container."""
+    def __init__(self, name, parameters=''):
+        """Constructor.
+
+        Args:
+            name (str): benchmark name.
+            parameters (str): benchmark parameters.
+        """
+        super().__init__(name, parameters)
+        self._platform_options = '--security-opt seccomp=unconfined --group-add video'

tests/benchmarks/docker_benchmarks/test_docker_base.py

@@ -5,8 +5,9 @@
 
 import re
 
+from tests.helper import decorator
 from superbench.benchmarks import BenchmarkType, ReturnCode
-from superbench.benchmarks.docker_benchmarks import DockerBenchmark
+from superbench.benchmarks.docker_benchmarks import DockerBenchmark, CudaDockerBenchmark, RocmDockerBenchmark
 
 
 class FakeDockerBenchmark(DockerBenchmark):
@@ -49,8 +50,61 @@ class FakeDockerBenchmark(DockerBenchmark):
         return True
 
 
+class FakeCudaDockerBenchmark(CudaDockerBenchmark):
+    """Fake benchmark inherit from CudaDockerBenchmark."""
+    def __init__(self, name, parameters=''):
+        """Constructor.
+
+        Args:
+            name: benchmark name.
+            parameters: benchmark parameters.
+        """
+        super().__init__(name, parameters)
+
+    def _process_raw_result(self, cmd_idx, raw_output):
+        """Function to process raw results and save the summarized results.
+
+          self._result.add_raw_data() and self._result.add_result() need to be called to save the results.
+
+        Args:
+            cmd_idx (int): the index of command corresponding with the raw_output.
+            raw_output (str): raw output string of the docker-benchmark.
+
+        Return:
+            True if the raw output string is valid and result can be extracted.
+        """
+        return True
+
+
+class FakeRocmDockerBenchmark(RocmDockerBenchmark):
+    """Fake benchmark inherit from RocmDockerBenchmark."""
+    def __init__(self, name, parameters=''):
+        """Constructor.
+
+        Args:
+            name: benchmark name.
+            parameters: benchmark parameters.
+        """
+        super().__init__(name, parameters)
+
+    def _process_raw_result(self, cmd_idx, raw_output):
+        """Function to process raw results and save the summarized results.
+
+          self._result.add_raw_data() and self._result.add_result() need to be called to save the results.
+
+        Args:
+            cmd_idx (int): the index of command corresponding with the raw_output.
+            raw_output (str): raw output string of the docker-benchmark.
+
+        Return:
+            True if the raw output string is valid and result can be extracted.
+        """
+        return True
+
+
+@decorator.cuda_test
 def test_docker_benchmark_base():
-    """Test MicroBenchmarkWithInvoke."""
+    """Test DockerBenchmark."""
     # Negative case - DOCKERBENCHMARK_IMAGE_NOT_SET.
     benchmark = FakeDockerBenchmark('fake')
     assert (benchmark._benchmark_type == BenchmarkType.DOCKER)
@@ -70,9 +124,21 @@ def test_docker_benchmark_base():
     assert (benchmark.run() is False)
     assert (benchmark.return_code == ReturnCode.DOCKERBENCHMARK_IMAGE_PULL_FAILURE)
 
-    # Test for DockerBenchmark._benchmark().
+    # Positive case
-    benchmark._commands.append("echo -n 'cost1: 10.2, cost2: 20.2'")
+    benchmark = FakeDockerBenchmark('fake')
-    benchmark._benchmark()
+    benchmark._image_uri = 'ubuntu'
-    assert (benchmark.raw_data['raw_output_0'] == ['cost1: 10.2, cost2: 20.2'])
+    benchmark._container_name = 'fake-docker-benchmark-test'
-    assert (benchmark.result['cost1'] == [10.2])
+    benchmark._entrypoint = 'echo'
-    assert (benchmark.result['cost2'] == [20.2])
+    benchmark._cmd = '-n "cost1: 10.2, cost2: 20.2"'
+    benchmark.run()
+    assert (
+        benchmark._commands[0] ==
+        'docker run -i --rm --privileged --net=host --ipc=host --name=fake-docker-benchmark-test'
+        '  --entrypoint echo ubuntu -n "cost1: 10.2, cost2: 20.2"'
+    )
+
+    # Test for _platform_options.
+    benchmark = FakeCudaDockerBenchmark('fake')
+    assert (benchmark._platform_options == '--gpus=all')
+    benchmark = FakeRocmDockerBenchmark('fake')
+    assert (benchmark._platform_options == '--security-opt seccomp=unconfined --group-add video')