Merge branch 'develop' into anfranc-nexilus/feature/add-notebook-params-and-debugger

.github/workflows/pythonpackage.yml

@@ -34,5 +34,5 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
-        pip install pytest
+        pip install pytest py4j
         pytest

README.md

@@ -319,6 +319,7 @@ FLAGS
     --notebook_params      Allows parameters to be passed from the CLI tool to the test notebook. From the 
                            notebook, these parameters can then be accessed by the notebook using 
                            the 'dbutils.widgets.get('key')' syntax.
+
 ```
 
 __Note:__ You can also use flags syntax for POSITIONAL ARGUMENTS

common/testresult.py

@@ -3,9 +3,13 @@ Copyright (c) Microsoft Corporation.
 Licensed under the MIT license.
 """
 
-from .pickleserializable import PickleSerializable
-import pickle
 import base64
+import pickle
+
+from py4j.protocol import Py4JJavaError
+
+from .pickleserializable import PickleSerializable
+
 
 def get_test_results():
     return TestResults()
@@ -30,6 +34,9 @@ class TestResults(PickleSerializable):
         self.total_execution_time = total_execution_time
 
     def serialize(self):
+        for i in self.results:
+            if isinstance(i.exception, Py4JJavaError):
+                i.exception = Exception(str(i.exception))
         bin_data = pickle.dumps(self)
         return str(base64.encodebytes(bin_data), "utf-8")
 

runtime/runner.py

@@ -0,0 +1,56 @@
+"""
+Copyright (c) Microsoft Corporation.
+
+Licensed under the MIT license.
+"""
+
+import common.scheduler as scheduler
+from common.testexecresults import TestExecResults
+from common.testresult import TestResults
+
+from runtime.nutterfixture import NutterFixture
+
+
+class NutterFixtureParallelRunner(object):
+    """Helper class to execute tests in parallel."""
+
+    def __init__(self, num_of_workers=1):
+        """Initialize the runner.
+
+        Args:
+            num_of_workers (int): number of parallel workers.
+        """
+        self.tests = []
+        self.num_of_workers = num_of_workers
+
+    def add_test_fixture(self, fixture):
+        """Add a test to the list of tests to run.
+
+        Args:
+            fixture (NutterFixture): the test to add.
+        """
+        if not isinstance(fixture, NutterFixture):
+            raise TypeError("fixture must be of type NutterFixture")
+        self.tests.append(fixture)
+
+    def execute(self):
+        """Execute the tests."""
+        sched = scheduler.get_scheduler(self.num_of_workers)
+
+        for i in self.tests:
+            sched.add_function(i.execute_tests)
+
+        results = sched.run_and_wait()
+
+        return self._collect_results(results)
+
+    def _collect_results(self, results):
+        """Collect all results in a single TestExecResults object."""
+        all_results = TestResults()
+
+        for funcres in results:
+            if funcres.func_result is not None:
+                for testres in funcres.func_result.test_results.results:
+                    all_results.append(testres)
+
+        return TestExecResults(all_results)

tests/nutter/test_testresult.py

@@ -3,11 +3,15 @@ Copyright (c) Microsoft Corporation.
 Licensed under the MIT license.
 """
 
-import pytest
-import json
-from common.testresult import TestResults, TestResult
-import pickle
 import base64
+import json
+import pickle
+
+import mock
+import pytest
+from common.testresult import TestResult, TestResults
+from py4j.protocol import Py4JError, Py4JJavaError
+
 
 def test__testresults_append__type_not_testresult__throws_error():
     # Arrange
@@ -74,6 +78,21 @@ def test__deserialize__invalid_pickle_data__throws_Exception():
     with pytest.raises(Exception):
         test_results.deserialize(invalid_pickle)
 
+def test__deserialize__p4jjavaerror__is_serializable_and_deserializable():
+    # Arrange
+    test_results = TestResults()
+
+    py4j_exception = get_mock_py4j_error_exception(get_mock_gateway_client(), mock_target_id="o123")
+
+    test_results.append(TestResult("Test Name", True, 1, [], py4j_exception))
+
+    with mock.patch('py4j.protocol.get_return_value') as mock_get_return_value:
+        mock_get_return_value.return_value = 'foo'
+        serialized_data = test_results.serialize()
+        deserialized_data = TestResults().deserialize(serialized_data)
+
+    assert test_results == deserialized_data
+
 
 def test__eq__test_results_equal_but_not_same_ref__are_equal():
     # Arrange
@@ -139,3 +158,25 @@ def test__deserialize__data_is_base64_str__can_deserialize():
     test_results_from_data = TestResults().deserialize(serialized_str)
 
     assert test_results == test_results_from_data
+
+
+def get_mock_gateway_client():
+    mock_client = mock.Mock()
+    mock_client.send_command.return_value = "0"
+    mock_client.converters = []
+    mock_client.is_connected.return_value = True
+    mock_client.deque = mock.Mock()
+    return mock_client
+
+
+def get_mock_java_object(mock_client, mock_target_id):
+    mock_java_object = mock.Mock()
+    mock_java_object._target_id = mock_target_id
+    mock_java_object._gateway_client = mock_client
+    return mock_java_object
+
+
+def get_mock_py4j_error_exception(mock_client, mock_target_id):
+    mock_java_object = get_mock_java_object(mock_client, mock_target_id)
+    mock_errmsg = "An error occurred while calling {}.load.".format(mock_target_id)
+    return Py4JJavaError(mock_errmsg, java_exception=mock_java_object)

tests/runtime/test_runner.py

@@ -0,0 +1,142 @@
+"""
+Copyright (c) Microsoft Corporation.
+
+Licensed under the MIT license.
+"""
+
+import time
+
+import pytest
+
+from runtime.nutterfixture import NutterFixture
+from runtime.runner import NutterFixtureParallelRunner
+
+test_cases = [
+    (1, 1),
+    (1, 2),
+    (2, 1),
+    (2, 2),
+    (3, 1),
+    (3, 2),
+    (3, 3)
+]
+
+@pytest.mark.parametrize('num_of_tests, num_of_workers', test_cases)
+def test__execute_tests__x_tests_x_workers__results_ok(num_of_tests, num_of_workers):
+    # Assemble list of tests
+    runner = NutterFixtureParallelRunner(num_of_workers)
+    for i in range(num_of_tests):
+        test_case = RunnerTestFixture()
+        runner.add_test_fixture(test_case)
+
+    # Execute tests
+    results = runner.execute()
+
+    # Assert results
+    assert len(results.test_results.results) == num_of_tests
+    assert results.test_results.passed() == True
+
+def test__execute_tests__3_tests_in_sequence_with_failed_assertion__results_ok():
+    # Arrange
+    tests = [
+        RunnerTestFixture(),
+        RunnerTestFixtureFailAssert(),
+        RunnerTestFixture()
+    ]
+
+    runner = NutterFixtureParallelRunner()
+    for i in tests:
+        runner.add_test_fixture(i)
+
+    # Act
+    results = runner.execute()
+
+    # Assert
+    assert len(results.test_results.results) == len(tests)
+    assert results.test_results.results[0].passed == True
+    assert results.test_results.results[1].passed == False
+    assert results.test_results.results[2].passed == True
+
+def test__execute_tests__3_tests_in_sequence_with_run_exception__results_ok():
+    # Arrange
+    tests = [
+        RunnerTestFixture(),
+        RunnerTestFixtureRunException(),
+        RunnerTestFixture()
+    ]
+
+    runner = NutterFixtureParallelRunner()
+    for i in tests:
+        runner.add_test_fixture(i)
+
+    # Act
+    results = runner.execute()
+
+    # Assert
+    assert len(results.test_results.results) == len(tests)
+    assert results.test_results.results[0].passed == True
+    assert results.test_results.results[1].passed == False
+    assert results.test_results.results[2].passed == True
+
+def test__execute_tests__3_tests_in_sequence_with_exec_exception__results_ok():
+    # Arrange
+    tests = [
+        RunnerTestFixture(),
+        RunnerTestFixtureExecuteException(),
+        RunnerTestFixture()
+    ]
+
+    runner = NutterFixtureParallelRunner()
+    for i in tests:
+        runner.add_test_fixture(i)
+
+    # Act
+    results = runner.execute()
+
+    # Assert
+    assert len(results.test_results.results) == len(tests) - 1
+    assert results.test_results.results[0].passed == True
+    assert results.test_results.results[1].passed == True
+
+class RunnerTestFixture(NutterFixture):
+    def before_test(self):
+        pass
+
+    def run_test(self):
+        pass
+
+    def assertion_test(self):
+        assert 1 == 1
+
+    def after_test(self):
+        pass
+
+class RunnerTestFixtureFailAssert(NutterFixture):
+    def before_test(self):
+        pass
+
+    def run_test(self):
+        pass
+
+    def assertion_test(self):
+        assert 1 != 1
+
+    def after_test(self):
+        pass
+
+class RunnerTestFixtureRunException(NutterFixture):
+    def before_test(self):
+        pass
+
+    def run_test(self):
+        raise(Exception())
+
+    def assertion_test(self):
+        assert 1 == 1
+
+    def after_test(self):
+        pass
+
+class RunnerTestFixtureExecuteException(NutterFixture):
+    def execute_tests(self):
+        raise(Exception())