Initial commit of runtime

runtime/fixtureloader.py

@@ -0,0 +1,82 @@
+from .testcase import TestCase
+
+
+def get_fixture_loader():
+    loader = FixtureLoader()
+    return loader
+
+
+class FixtureLoader():
+    def __init__(self):
+        self.__test_case_dictionary = {}
+        pass
+
+    def load_fixture(self, nutter_fixture):
+        if nutter_fixture is None:
+            raise ValueError("Must pass NutterFixture")
+
+        all_attributes = dir(nutter_fixture)
+        for attribute in all_attributes:
+            is_test_method = self.__is_test_method(attribute)
+            if is_test_method:
+                test_full_name = attribute
+                test_name = self.__get_test_name(attribute)
+                func = getattr(nutter_fixture, test_full_name)
+                if func is None:
+                    continue
+
+                if test_name == "before_all" or test_name == "after_all":
+                    continue
+
+                test_case = None
+                if test_name in self.__test_case_dictionary:
+                    test_case = self.__test_case_dictionary[test_name]
+
+                if test_case is None:
+                    test_case = TestCase(test_name)
+
+                test_case = self.__set_method(test_case, test_full_name, func)
+
+                self.__test_case_dictionary[test_name] = test_case
+
+        return self.__test_case_dictionary
+
+    def __is_test_method(self, attribute):
+        if attribute.startswith("before_") or \
+                attribute.startswith("run_") or \
+                attribute.startswith("assertion_") or \
+                attribute.startswith("after_"):
+            return True
+        return False
+
+    def __set_method(self, case, name, func):
+        if name.startswith("before_"):
+            case.set_before(func)
+            return case
+        if name.startswith("run_"):
+            case.set_run(func)
+            return case
+        if name.startswith("assertion_"):
+            case.set_assertion(func)
+            return case
+        if name.startswith("after_"):
+            case.set_after(func)
+            return case
+
+        return case
+
+    def __get_test_name(self, full_name):
+        if full_name == "before_all" or full_name == "after_all":
+            return full_name
+
+        name = self.__remove_prefix(full_name, "before_")
+        name = self.__remove_prefix(name, "run_")
+        name = self.__remove_prefix(name, "assertion_")
+        name = self.__remove_prefix(name, "after_")
+
+        return name
+
+    def __remove_prefix(self, text, prefix):
+        if text.startswith(prefix):
+            return text[len(prefix):]
+        return text

runtime/nutterfixture.py

@@ -0,0 +1,70 @@
+import json
+import logging
+from abc import abstractmethod, ABCMeta
+from common.testresult import TestResults
+from .fixtureloader import FixtureLoader
+from common.testexecresults import TestExecResults
+
+
+def tag(the_tag):
+    def tag_decorator(function):
+        if isinstance(the_tag, list) == False and isinstance(the_tag, str) == False:
+            raise ValueError("the_tag must be a string or a list")
+        if str.startswith(function.__name__, "run_") == False:
+            raise ValueError("a tag may only decorate a run_ method")
+
+        function.tag = the_tag
+        return function
+    return tag_decorator
+
+
+class NutterFixture(object):
+    """
+    """
+
+    __metaclass__ = ABCMeta
+
+    def __init__(self):
+        self.data_loader = FixtureLoader()
+        self.test_results = TestResults()
+        self._logger = logging.getLogger('NutterRunner')
+
+    def execute_tests(self):
+        self.__load_fixture()
+
+        if len(self.__test_case_dict) > 0 and self.__has_method("before_all"):
+            logging.debug('Running before_all()')
+            self.before_all()
+
+        for key, value in self.__test_case_dict.items():
+            logging.debug('Running test: {}'.format(key))
+            test_result = value.execute_test()
+            logging.debug('Completed running test: {}'.format(key))
+            self.test_results.append(test_result)
+
+        if len(self.__test_case_dict) > 0 and self.__has_method("after_all"):
+            logging.debug('Running after_all()')
+            self.after_all()
+
+        return TestExecResults(self.test_results)
+
+    def __load_fixture(self):
+        test_case_dict = self.data_loader.load_fixture(self)
+        if test_case_dict is None:
+            logging.fatal("Invalid Test Fixture")
+            raise InvalidTestFixtureException("Invalid Test Fixture")
+        self.__test_case_dict = test_case_dict
+
+        logging.debug("Found {} test cases".format(len(test_case_dict)))
+        for key, value in self.__test_case_dict.items():
+            logging.debug('Test Case: {}'.format(key))
+
+    def __has_method(self, method_name):
+        method = getattr(self, method_name, None)
+        if callable(method):
+            return True
+        return False
+
+
+class InvalidTestFixtureException(Exception):
+    pass

runtime/testcase.py

@@ -0,0 +1,99 @@
+import os
+import sys
+import time
+import traceback
+from common.testresult import TestResult
+
+
+def get_testcase(test_name):
+
+    tc = TestCase(test_name)
+
+    return tc
+
+
+class TestCase():
+    ERROR_MESSAGE_RUN_MISSING = "TestCase does not contain a run function.  Please pass a function to set_run"
+    ERROR_MESSAGE_ASSERTION_MISSING = "TestCase does not contain an assertion function.  Please pass a function to set_assertion"
+
+    def __init__(self, test_name):
+        self.test_name = test_name
+        self.before = None
+        self.__before_set = False
+        self.run = None
+        self.assertion = None
+        self.after = None
+        self.__after_set = False
+        self.invalid_message = ""
+        self.tags = []
+
+    def set_before(self, before):
+        self.before = before
+        self.__before_set = True
+
+    def set_run(self, run):
+        self.run = run
+
+    def set_assertion(self, assertion):
+        self.assertion = assertion
+
+    def set_after(self, after):
+        self.after = after
+        self.__after_set = True
+
+    def execute_test(self):
+        start_time = time.perf_counter()
+        try:
+            if hasattr(self.run, "tag"):
+                if isinstance(self.run.tag, list):
+                    self.tags.extend(self.run.tag)
+                else:
+                    self.tags.append(self.run.tag)
+            if self.is_valid() == False:
+                raise NoTestCasesFoundError(
+                    "Both a run and an assertion are required for every test")
+            if self.__before_set == True and self.before is not None:
+                self.before()
+            self.run()
+            self.assertion()
+            if self.__after_set == True and self.after is not None:
+                self.after()
+
+        except Exception as exc:
+            stack = traceback.print_exc()
+            return TestResult(self.test_name, False, self.__get_elapsed_time(start_time), self.tags, exc, traceback.format_exc())
+
+        return TestResult(self.test_name, True, self.__get_elapsed_time(start_time), self.tags, None)
+
+    def is_valid(self):
+        is_valid = True
+
+        if self.run is None:
+            self.__add_message_to_error(self.ERROR_MESSAGE_RUN_MISSING)
+            is_valid = False
+
+        if self.assertion is None:
+            self.__add_message_to_error(self.ERROR_MESSAGE_ASSERTION_MISSING)
+            is_valid = False
+
+        return is_valid
+
+    def __get_elapsed_time(self, start_time):
+        end_time = time.perf_counter()
+        elapsed_time = end_time - start_time
+        return elapsed_time
+
+    def __add_message_to_error(self, message):
+        if self.invalid_message:
+            self.invalid_message += os.linesep
+
+        self.invalid_message += message
+
+    def get_invalid_message(self):
+        self.is_valid()
+
+        return self.invalid_message
+
+
+class NoTestCasesFoundError(Exception):
+    pass

tests/cli/test_eventhandlers.py

@@ -0,0 +1,134 @@
+import pytest
+from queue import Queue
+from cli.eventhandlers import ConsoleEventHandler
+from common.api import NutterStatusEvents, ExecutionResultEventData
+from common.statuseventhandler import StatusEvent
+
+
+def test__handle__nutterstatusevents_testslisting__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    path = '/path'
+    events = [StatusEvent(NutterStatusEvents.TestsListing, path)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper('Looking for tests in {}'.format(path))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def test__handle__nutterstatusevents_testsexecutionrequest__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    pattern = '/path'
+    events = [StatusEvent(NutterStatusEvents.TestExecutionRequest, pattern)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper('Execution request: {}'.format(pattern))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def test__handle__nutterstatusevents_testslistingfiltered__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    num_of_tests = 1
+    events = [StatusEvent(
+        NutterStatusEvents.TestsListingFiltered, num_of_tests)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper(
+        '{} tests matched the pattern'.format(num_of_tests))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def test__handle__nutterstatusevents_testlistingresults__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    num_of_tests = 1
+    events = [StatusEvent(
+        NutterStatusEvents.TestsListingResults, num_of_tests)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper('{} tests found'.format(num_of_tests))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def test__handle__nutterstatusevents_testscheduling__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    num_of_tests = 1
+    console_event_handler._filtered_tests = num_of_tests
+    events = [StatusEvent(NutterStatusEvents.TestScheduling, num_of_tests)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper(
+        '{} of {} tests scheduled for execution'.format(num_of_tests, num_of_tests))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def test__handle__nutterstatusevents_testsexecutionresult__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    num_of_tests = 1
+    done_tests = 1
+    console_event_handler._listed_tests = num_of_tests
+    events = [StatusEvent(
+        NutterStatusEvents.TestExecutionResult, num_of_tests)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper(
+        '{} of {} tests executed.'.format(done_tests, num_of_tests))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def test__handle__nutterstatusevents_testsexecutionresult__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    num_of_tests = 1
+    done_tests = 1
+    console_event_handler._listed_tests = num_of_tests
+    events = [StatusEvent(
+        NutterStatusEvents.TestExecutionResult, num_of_tests)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper(
+        '{} of {} tests executed'.format(done_tests, num_of_tests))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def test__handle__nutterstatusevents_testexecuted__output_is_valid(mocker):
+    console_event_handler = ConsoleEventHandler(False)
+    mocker.patch.object(console_event_handler, '_print_output')
+    event_data = ExecutionResultEventData('/my', True, 'http://url')
+    events = [StatusEvent(NutterStatusEvents.TestExecuted, event_data)]
+    queue = _get_queue_with_events(events)
+
+    console_event_handler._get_and_handle(queue)
+
+    expected = _get_output_wrapper('{} Success:{} {}'.format(
+        event_data.notebook_path, event_data.success, event_data.notebook_run_page_url))
+    console_event_handler._print_output.assert_called_with(expected)
+
+
+def _get_output_wrapper(output):
+    return '--> {}\n'.format(output)
+
+
+def _get_queue_with_events(events):
+    queue = Queue()
+    for event in events:
+        queue.put(event)
+    return queue

tests/cli/test_nuttercli.py

@@ -0,0 +1,203 @@
+import pytest
+import os
+import json
+import cli.nuttercli as nuttercli
+from cli.nuttercli import NutterCLI
+from common.apiclientresults import ExecuteNotebookResult
+import mock
+from common.testresult import TestResults, TestResult
+from cli.reportsman import ReportWriterManager, ReportWritersTypes, ReportWriters
+
+
+def test__get_cli_version__without_build__env_var__returns_value():
+    version = nuttercli.get_cli_version()
+    assert version is not None
+
+
+def test__get_cli_header_value():
+    version = nuttercli.get_cli_version()
+    header = 'Nutter Version {}\n'.format(version)
+    header += '+' * 50
+    header += '\n'
+
+    assert nuttercli.get_cli_header() == header
+
+
+
+def test__get_cli_version__with_build__env_var__returns_value(mocker):
+    version = nuttercli.get_cli_version()
+    build_number = '1.2.3'
+    mocker.patch.dict(
+        os.environ, {nuttercli.BUILD_NUMBER_ENV_VAR: build_number})
+    version_with_build_number = nuttercli.get_cli_version()
+    assert version_with_build_number == '{}.{}'.format(version, build_number)
+
+def test__get_version_label__valid_string(mocker):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': 'myhost'})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': 'mytoken'})
+
+    version = nuttercli.get_cli_version()
+    expected = 'Nutter Version {}'.format(version)
+    cli = NutterCLI()
+    version_from_cli =  cli._get_version_label()
+
+    assert expected == version_from_cli
+
+
+def test__nutter_cli_ctor__handles__version_and_exits_0(mocker):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': 'myhost'})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': 'mytoken'})
+
+
+    with pytest.raises(SystemExit) as mock_ex:
+        cli = NutterCLI(version=True)
+
+    assert mock_ex.type == SystemExit
+    assert mock_ex.value.code == 0
+
+def test__run__pattern__display_results(mocker):
+    test_results = TestResults().serialize()
+    cli = _get_cli_for_tests(
+        mocker, 'SUCCESS', 'TERMINATED', test_results)
+
+    mocker.patch.object(cli, '_display_test_results')
+    cli.run('my*', 'cluster')
+    assert cli._display_test_results.call_count == 1
+
+
+def test__nutter_cli_ctor__handles__configurationexception_and_exits_1(mocker):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': ''})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': ''})
+
+    with pytest.raises(SystemExit) as mock_ex:
+        cli = NutterCLI()
+
+    assert mock_ex.type == SystemExit
+    assert mock_ex.value.code == 1
+
+
+def test__run__one_test_fullpath__display_results(mocker):
+    test_results = TestResults().serialize()
+    cli = _get_cli_for_tests(
+        mocker, 'SUCCESS', 'TERMINATED', test_results)
+
+    mocker.patch.object(cli, '_display_test_results')
+    cli.run('test_mynotebook2', 'cluster')
+    assert cli._display_test_results.call_count == 1
+
+def test__run_one_test_junit_writter__writer_writes(mocker):
+    test_results = TestResults().serialize()
+    cli = _get_cli_for_tests(
+        mocker, 'SUCCESS', 'TERMINATED', test_results)
+    mocker.patch.object(cli, '_get_report_writer_manager')
+    mock_report_manager = ReportWriterManager(ReportWriters.JUNIT)
+    mocker.patch.object(mock_report_manager, 'write')
+    mocker.patch.object(mock_report_manager, 'add_result')
+
+    cli._get_report_writer_manager.return_value = mock_report_manager
+
+    cli.run('test_mynotebook2', 'cluster')
+
+    assert mock_report_manager.add_result.call_count == 1
+    assert mock_report_manager.write.call_count == 1
+    assert not mock_report_manager._providers[ReportWritersTypes.JUNIT].has_data(
+    )
+
+
+def test__list__none__display_result(mocker):
+    cli = _get_cli_for_tests(
+        mocker, 'SUCCESS', 'TERMINATED', 'IHAVERETURNED')
+
+    mocker.patch.object(cli, '_display_list_results')
+    cli.list('/')
+    assert cli._display_list_results.call_count == 1
+
+
+def _get_cli_for_tests(mocker, result_state, life_cycle_state, notebook_result):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': 'myhost'})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': 'mytoken'})
+    cli = NutterCLI()
+    mocker.patch.object(cli._nutter, 'run_test')
+    cli._nutter.run_test.return_value = _get_run_test_response(
+        result_state, life_cycle_state, notebook_result)
+    mocker.patch.object(cli._nutter, 'run_tests')
+    cli._nutter.run_tests.return_value = _get_run_tests_response(
+        result_state, life_cycle_state, notebook_result)
+    mocker.patch.object(cli._nutter, 'list_tests')
+    cli._nutter.list_tests.return_value = _get_list_tests_response()
+
+    return cli
+
+
+def _get_run_test_response(result_state, life_cycle_state, notebook_result):
+    data_json = """
+                {"notebook_output":
+                {"result": "IHaveReturned", "truncated": false},
+                "metadata":
+                {"execution_duration": 15000,
+                "run_type": "SUBMIT_RUN",
+                "cleanup_duration": 0,
+                "number_in_job": 1,
+                "cluster_instance":
+                {"cluster_id": "0925-141d1222-narcs242",
+                "spark_context_id": "803963628344534476"},
+                "creator_user_name": "abc@microsoft.com",
+                "task": {"notebook_task": {"notebook_path": "/test_mynotebook"}},
+                "run_id": 7, "start_time": 1569887259173,
+                "job_id": 4,
+                "state": {"result_state": "SUCCESS", "state_message": "",
+                "life_cycle_state": "TERMINATED"}, "setup_duration": 2000,
+                "run_page_url": "https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1",
+                "cluster_spec": {"existing_cluster_id": "0925-141122-narcs242"}, "run_name": "myrun"}}
+                """
+    data_dict = json.loads(data_json)
+    data_dict['notebook_output']['result'] = notebook_result
+    data_dict['metadata']['state']['result_state'] = result_state
+    data_dict['metadata']['state']['life_cycle_state'] = life_cycle_state
+
+    return ExecuteNotebookResult.from_job_output(data_dict)
+
+
+def _get_list_tests_response():
+    result = {}
+    result['test_mynotebook'] = '/test_mynotebook'
+    result['test_mynotebook2'] = '/test_mynotebook2'
+    return result
+
+
+def _get_run_tests_response(result_state, life_cycle_state, notebook_result):
+    data_json = """
+                {"notebook_output":
+                {"result": "IHaveReturned", "truncated": false},
+                "metadata":
+                {"execution_duration": 15000,
+                "run_type": "SUBMIT_RUN",
+                "cleanup_duration": 0,
+                "number_in_job": 1,
+                "cluster_instance":
+                {"cluster_id": "0925-141d1222-narcs242",
+                "spark_context_id": "803963628344534476"},
+                "creator_user_name": "abc@microsoft.com",
+                "task": {"notebook_task": {"notebook_path": "/test_mynotebook"}},
+                "run_id": 7, "start_time": 1569887259173,
+                "job_id": 4,
+                "state": {"result_state": "SUCCESS", "state_message": "",
+                "life_cycle_state": "TERMINATED"}, "setup_duration": 2000,
+                "run_page_url": "https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1",
+                "cluster_spec": {"existing_cluster_id": "0925-141122-narcs242"}, "run_name": "myrun"}}
+                """
+    data_dict = json.loads(data_json)
+    data_dict['notebook_output']['result'] = notebook_result
+    data_dict['metadata']['state']['result_state'] = result_state
+    data_dict['metadata']['state']['life_cycle_state'] = life_cycle_state
+
+    data_dict2 = json.loads(data_json)
+    data_dict2['notebook_output']['result'] = notebook_result
+    data_dict2['metadata']['state']['result_state'] = result_state
+    data_dict2['metadata']['task']['notebook_task']['notebook_path'] = '/test_mynotebook2'
+    data_dict2['metadata']['state']['life_cycle_state'] = life_cycle_state
+
+    results = []
+    results.append(ExecuteNotebookResult.from_job_output(data_dict))
+    results.append(ExecuteNotebookResult.from_job_output(data_dict2))
+    return results

tests/cli/test_reportsman.py

@@ -0,0 +1,75 @@
+import pytest
+from common.testresult import TestResults, TestResult
+from common.resultreports import JunitXMLReportWriter
+from common.resultreports import TagsReportWriter
+from cli.reportsman import ReportWriterManager, ReportWriters, ReportWritersTypes
+import common.api as nutter_api
+
+def test__reportwritermanager_ctor__junit_report__valid_manager():
+    report_writer_man = ReportWriterManager(ReportWriters.JUNIT)
+
+    assert len(report_writer_man._providers) == 1
+    report_man = report_writer_man._providers[ReportWritersTypes.JUNIT]
+    assert isinstance(report_man, JunitXMLReportWriter)
+
+def test__reportwritermanager_ctor__tags_report__valid_manager():
+    report_writer_man = ReportWriterManager(ReportWriters.TAGS)
+
+    assert len(report_writer_man._providers) == 1
+    report_man = report_writer_man._providers[ReportWritersTypes.TAGS]
+    assert isinstance(report_man, TagsReportWriter)
+
+
+def test__reportwritermanager_ctor__tags_and_junit_report__valid_manager():
+    report_writer_man = ReportWriterManager(ReportWriters.TAGS + ReportWriters.JUNIT)
+
+    assert len(report_writer_man._providers) == 2
+    report_man = report_writer_man._providers[ReportWritersTypes.TAGS]
+    assert isinstance(report_man, TagsReportWriter)
+    report_man = report_writer_man._providers[ReportWritersTypes.JUNIT]
+    assert isinstance(report_man, JunitXMLReportWriter)
+
+
+def test__reportwritermanager_ctor__invalid_report__empty_manager():
+    report_writer_man = ReportWriterManager(0)
+
+    assert len(report_writer_man._providers) == 0
+
+def test__add_result__junit_provider_one_test_result__provider_has_data():
+    report_writer_man = ReportWriterManager(ReportWriters.JUNIT)
+    test_results = TestResults()
+    test_results.append(TestResult("mycase", True, 10, []))
+    report_writer_man.add_result('notepad', test_results)
+
+    report_man = report_writer_man._providers[ReportWritersTypes.JUNIT]
+    assert isinstance(report_man, JunitXMLReportWriter)
+    assert report_man.has_data()
+
+
+def test__add_result__junit_provider_zero_test_result__provider_has_data():
+    report_writer_man = ReportWriterManager(ReportWriters.JUNIT)
+    test_results = TestResults()
+    report_writer_man.add_result('notepad', test_results)
+
+    report_man = report_writer_man._providers[ReportWritersTypes.JUNIT]
+    assert report_man.has_data()
+
+def test__add_result__tags_provider_one_test_result__provider_has_data():
+    report_writer_man = ReportWriterManager(ReportWriters.TAGS)
+    test_results = TestResults()
+    test_results.append(TestResult("mycase", True, 10, ['hello']))
+    report_writer_man.add_result('notepad', test_results)
+
+    report_man = report_writer_man._providers[ReportWritersTypes.TAGS]
+    assert report_man.has_data()
+
+
+def test__write__two_providers__returns_two_names():
+    report_writer_man = ReportWriterManager(ReportWriters.TAGS + ReportWriters.JUNIT)
+    test_results = TestResults()
+    test_results.append(TestResult("mycase", True, 10, ['hello']))
+    report_writer_man.add_result('notepad', test_results)
+
+    results = report_writer_man.providers_names()
+
+    assert len(results) == 2

tests/cli/test_resultsvalidator.py

@@ -0,0 +1,164 @@
+import pytest
+import common.testresult as testresult
+from common.apiclientresults import ExecuteNotebookResult
+from cli.resultsvalidator import ExecutionResultsValidator, TestCaseFailureException, JobExecutionFailureException, NotebookExecutionFailureException, InvalidNotebookOutputException
+import json
+
+
+def test__validate__results_is_none__valueerror():
+    with pytest.raises(ValueError):
+        ExecutionResultsValidator().validate(None)
+
+
+def test__validate__results_are_empty__no_ex():
+    exec_results = []
+    ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_have_no_testcases__no_ex():
+    test_results = testresult.TestResults()
+    exec_result = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+    exec_results = [exec_result]
+
+    ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_have_one_testcases__no_ex():
+    test_results = testresult.TestResults()
+    test_case = testresult.TestResult(
+        test_name="mytest_case", passed=True, execution_time=1, tags = [])
+    test_results.append(test_case)
+
+    exec_result = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+    exec_results = [exec_result]
+
+    ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_have_two_exec_results__no_ex():
+    test_results = testresult.TestResults()
+    test_case = testresult.TestResult(
+        test_name="mytest_case", passed=True, execution_time=1, tags = [])
+    test_results.append(test_case)
+
+    exec_result = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+    exec_results = [exec_result, exec_result]
+
+    ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_have_two_testcases__no_ex():
+    test_results = testresult.TestResults()
+    test_case = testresult.TestResult(
+        test_name="mytest_case", passed=True, execution_time=1, tags = [])
+    test_results.append(test_case)
+    test_case = testresult.TestResult(
+        test_name="mytest2_case", passed=True, execution_time=1, tags = [])
+    test_results.append(test_case)
+
+    exec_result = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+    exec_results = [exec_result]
+
+    ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_have_two_testcases_one_failure__no_ex():
+    test_results = testresult.TestResults()
+    test_case = testresult.TestResult(
+        test_name="mytest_case", passed=True, execution_time=1, tags = [])
+    test_results.append(test_case)
+    test_case = testresult.TestResult(
+        test_name="mytest2_case", passed=False, execution_time=1, tags = [])
+    test_results.append(test_case)
+
+    exec_result = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+    exec_results = [exec_result]
+
+    with pytest.raises(TestCaseFailureException):
+        ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_have_failed_testcase__throws_testcasefailurexception():
+    test_results = testresult.TestResults()
+    test_case = testresult.TestResult(
+        test_name="mytest_case", passed=False, execution_time=1, tags = [])
+    test_results.append(test_case)
+
+    exec_result = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+    exec_results = [exec_result]
+
+    with pytest.raises(TestCaseFailureException):
+        ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_have_invalid_output__throws_invalidnotebookoutputexception():
+
+    exec_result = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', '')
+    exec_results = [exec_result]
+
+    with pytest.raises(InvalidNotebookOutputException):
+        ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_with_notebook_failure__throws_notebookexecutionfailureexception():
+    test_results = testresult.TestResults()
+    test_case = testresult.TestResult(
+        test_name="mytest_case", passed=False, execution_time=1, tags = [])
+    test_results.append(test_case)
+
+    exec_result = __get_ExecuteNotebookResult(
+        'FAILED', 'TERMINATED', test_results.serialize())
+    exec_results = [exec_result]
+
+    with pytest.raises(NotebookExecutionFailureException):
+        ExecutionResultsValidator().validate(exec_results)
+
+
+def test__validate__results_with_job_failure__throws_jobexecutionfailureexception():
+    test_results = testresult.TestResults()
+    test_case = testresult.TestResult(
+        test_name="mytest_case", passed=False, execution_time=1, tags = [])
+    test_results.append(test_case)
+
+    exec_result = __get_ExecuteNotebookResult(
+        'FAILED', 'INTERNAL_ERROR', test_results.serialize())
+    exec_results = [exec_result]
+
+    with pytest.raises(JobExecutionFailureException):
+        ExecutionResultsValidator().validate(exec_results)
+
+
+def __get_ExecuteNotebookResult(result_state, life_cycle_state, notebook_result):
+    data_json = """
+                {"notebook_output":
+                {"result": "IHaveReturned", "truncated": false},
+                "metadata":
+                {"execution_duration": 15000,
+                "run_type": "SUBMIT_RUN",
+                "cleanup_duration": 0,
+                "number_in_job": 1,
+                "cluster_instance":
+                {"cluster_id": "0925-141d1222-narcs242",
+                "spark_context_id": "803963628344534476"},
+                "creator_user_name": "abc@microsoft.com",
+                "task": {"notebook_task": {"notebook_path": "/test_mynotebook"}},
+                "run_id": 7, "start_time": 1569887259173,
+                "job_id": 4,
+                "state": {"result_state": "SUCCESS", "state_message": "",
+                "life_cycle_state": "TERMINATED"}, "setup_duration": 2000,
+                "run_page_url": "https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1",
+                "cluster_spec": {"existing_cluster_id": "0925-141122-narcs242"}, "run_name": "myrun"}}
+                """
+    data_dict = json.loads(data_json)
+    data_dict['notebook_output']['result'] = notebook_result
+    data_dict['metadata']['state']['result_state'] = result_state
+    data_dict['metadata']['state']['life_cycle_state'] = life_cycle_state
+
+    return ExecuteNotebookResult.from_job_output(data_dict)

tests/databricks/test_apiclient.py

@@ -0,0 +1,217 @@
+import pytest
+from common import apiclient as client
+from common.apiclient import DatabricksAPIClient
+import os
+import json
+
+
+def test__databricks_client__token_host_notset__clientfails(mocker):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': ''})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': ''})
+
+    with pytest.raises(client.InvalidConfigurationException):
+        dbclient = client.databricks_client()
+
+
+def test__databricks_client__token_host_set__clientreturns(mocker):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': 'myhost'})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': 'mytoken'})
+
+    dbclient = client.databricks_client()
+
+    assert isinstance(dbclient, DatabricksAPIClient)
+
+
+def test__list_notebooks__onenotebook__okay(mocker):
+    db = __get_client(mocker)
+    mocker.patch.object(db.inner_dbclient.workspace, 'list')
+
+    objects = """{"objects":[
+    {"object_type":"NOTEBOOK","path":"/nutfixjob","language":"PYTHON"},
+    {"object_type":"DIRECTORY","path":"/ETL-Part-3-1.0.3"}]}"""
+
+    db.inner_dbclient.workspace.list.return_value = json.loads(objects)
+
+    notebooks = db.list_notebooks('/')
+
+    assert len(notebooks) == 1
+
+
+def test__list_notebooks__zeronotebook__okay(mocker):
+    db = __get_client(mocker)
+    mocker.patch.object(db.inner_dbclient.workspace, 'list')
+
+    objects = """{"objects":[
+    {"object_type":"DIRECTORY","path":"/ETL-Part-3-1.0.3"}]}"""
+
+    db.inner_dbclient.workspace.list.return_value = json.loads(objects)
+
+    notebooks = db.list_notebooks('/')
+
+    assert len(notebooks) == 0
+
+
+def test__execute_notebook__emptypath__valueerrror(mocker):
+    db = __get_client(mocker)
+
+    with pytest.raises(ValueError):
+        db.execute_notebook('', 'cluster')
+
+
+def test__execute_notebook__nonepath__valueerror(mocker):
+    db = __get_client(mocker)
+
+    with pytest.raises(ValueError):
+        db.execute_notebook(None, 'cluster')
+
+
+def test__execute_notebook__emptycluster__valueerror(mocker):
+    db = __get_client(mocker)
+
+    with pytest.raises(ValueError):
+        db.execute_notebook('/', '')
+
+
+def test__execute_notebook__non_dict_params__valueerror(mocker):
+    db = __get_client(mocker)
+
+    with pytest.raises(ValueError):
+        db.execute_notebook('/', 'cluster', notebook_params='')
+
+
+def test__execute_notebook__nonecluster__valueerror(mocker):
+    db = __get_client(mocker)
+
+    with pytest.raises(ValueError):
+        db.execute_notebook('/', None)
+
+
+def test__execute_notebook__success__executeresult_has_run_url(mocker):
+    run_page_url = "http://runpage"
+    output_data = __get_submit_run_response(
+        'SUCCESS', 'TERMINATED', '', run_page_url)
+    run_id = {}
+    run_id['run_id'] = 1
+    db = __get_client_for_execute_notebook(mocker, output_data, run_id)
+
+    result = db.execute_notebook('/mynotebook', 'clusterid')
+
+    assert result.notebook_run_page_url == run_page_url
+
+def test__execute_notebook__failure__executeresult_has_run_url(mocker):
+    run_page_url = "http://runpage"
+    output_data = __get_submit_run_response(
+        'FAILURE', 'TERMINATED', '', run_page_url)
+    run_id = {}
+    run_id['run_id'] = 1
+    db = __get_client_for_execute_notebook(mocker, output_data, run_id)
+
+    result = db.execute_notebook('/mynotebook', 'clusterid')
+
+    assert result.notebook_run_page_url == run_page_url
+
+
+def test__execute_notebook__terminatestate__success(mocker):
+    output_data = __get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    run_id = {}
+    run_id['run_id'] = 1
+    db = __get_client_for_execute_notebook(mocker, output_data, run_id)
+
+    result = db.execute_notebook('/mynotebook', 'clusterid')
+
+    assert result.task_result_state == 'TERMINATED'
+
+
+def test__execute_notebook__skippedstate__resultstate_is_SKIPPED(mocker):
+    output_data = __get_submit_run_response('', 'SKIPPED', '')
+    run_id = {}
+    run_id['run_id'] = 1
+    db = __get_client_for_execute_notebook(mocker, output_data, run_id)
+
+    result = db.execute_notebook('/mynotebook', 'clusterid')
+
+    assert result.task_result_state == 'SKIPPED'
+
+
+def test__execute_notebook__internal_error_state__resultstate_is_INTERNAL_ERROR(mocker):
+    output_data = __get_submit_run_response('', 'INTERNAL_ERROR', '')
+    run_id = {}
+    run_id['run_id'] = 1
+    db = __get_client_for_execute_notebook(mocker, output_data, run_id)
+
+    result = db.execute_notebook('/mynotebook', 'clusterid')
+
+    assert result.task_result_state == 'INTERNAL_ERROR'
+
+
+def test__execute_notebook__timeout_1_sec_lcs_isrunning__timeoutexception(mocker):
+    output_data = __get_submit_run_response('', 'RUNNING', '')
+    run_id = {}
+    run_id['run_id'] = 1
+    db = __get_client_for_execute_notebook(mocker, output_data, run_id)
+
+    with pytest.raises(client.TimeOutException):
+        db.min_timeout = 1
+        result = db.execute_notebook('/mynotebook', 'clusterid', timeout=1)
+
+
+def test__execute_notebook__timeout_greater_than_min__valueerror(mocker):
+    output_data = __get_submit_run_response('', 'RUNNING', '')
+    run_id = {}
+    run_id['run_id'] = 1
+    db = __get_client_for_execute_notebook(mocker, output_data, run_id)
+
+    with pytest.raises(ValueError):
+        db.min_timeout = 10
+        result = db.execute_notebook('/mynotebook', 'clusterid', timeout=1)
+
+
+default_run_page_url = 'https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1'
+
+
+def __get_submit_run_response(task_result_state, life_cycle_state, result, run_page_url=default_run_page_url):
+    data_json = """
+                {"notebook_output":
+                {"result": "IHaveReturned", "truncated": false},
+                "metadata":
+                {"execution_duration": 15000,
+                "run_type": "SUBMIT_RUN",
+                "cleanup_duration": 0,
+                "number_in_job": 1,
+                "cluster_instance":
+                {"cluster_id": "0925-141d1222-narcs242",
+                "spark_context_id": "803963628344534476"},
+                "creator_user_name": "abc@microsoft.com",
+                "task": {"notebook_task": {"notebook_path": "/mynotebook"}},
+                "run_id": 7, "start_time": 1569887259173,
+                "job_id": 4,
+                "state": {"result_state": "SUCCESS", "state_message": "",
+                "life_cycle_state": "TERMINATED"}, "setup_duration": 2000,
+                "run_page_url": "https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1",
+                "cluster_spec": {"existing_cluster_id": "0925-141122-narcs242"}, "run_name": "myrun"}}
+                """
+    data_dict = json.loads(data_json)
+    data_dict['notebook_output']['result'] = result
+    data_dict['metadata']['state']['result_state'] = task_result_state
+    data_dict['metadata']['state']['life_cycle_state'] = life_cycle_state
+    data_dict['metadata']['run_page_url'] = run_page_url
+
+    return json.dumps(data_dict)
+
+
+def __get_client_for_execute_notebook(mocker, output_data, run_id):
+    db = __get_client(mocker)
+    mocker.patch.object(db.inner_dbclient.jobs, 'submit_run')
+    db.inner_dbclient.jobs.submit_run.return_value = run_id
+    mocker.patch.object(db.inner_dbclient.jobs, 'get_run_output')
+    db.inner_dbclient.jobs.get_run_output.return_value = json.loads(
+        output_data)
+
+    return db
+
+
+def __get_client(mocker):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': 'myhost'})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': 'mytoken'})
+
+    return DatabricksAPIClient()

tests/databricks/test_authconfig.py

@@ -0,0 +1,50 @@
+import pytest
+import os
+from common import authconfig as auth
+
+def test_tokenhostset_okay(mocker):
+    mocker.patch.dict(os.environ,{'DATABRICKS_HOST':'host'})
+    mocker.patch.dict(os.environ,{'DATABRICKS_TOKEN':'token'})
+
+    config = auth.get_auth_config()
+    # Assert
+    assert config != None
+    assert config.host == 'host'
+    assert config.token == 'token'
+
+def test_onlytokenset_none(mocker):
+    mocker.patch.dict(os.environ,{'DATABRICKS_HOST':''})
+    mocker.patch.dict(os.environ,{'DATABRICKS_TOKEN':'token'})
+
+    config = auth.get_auth_config()
+    # Assert
+    assert config == None
+
+def test_tokenhostsetemtpy_none(mocker):
+    mocker.patch.dict(os.environ,{'DATABRICKS_HOST':''})
+    mocker.patch.dict(os.environ,{'DATABRICKS_TOKEN':''})
+
+    config = auth.get_auth_config()
+    # Assert
+    assert config == None
+
+def test_onlyhostset_none(mocker):
+    mocker.patch.dict(os.environ,{'DATABRICKS_HOST':'host'})
+    mocker.patch.dict(os.environ,{'DATABRICKS_TOKEN':''})
+
+    config = auth.get_auth_config()
+    # Assert
+    assert config == None
+
+def test_tokenhostinsecureset_okay(mocker):
+    mocker.patch.dict(os.environ,{'DATABRICKS_HOST':'host'})
+    mocker.patch.dict(os.environ,{'DATABRICKS_TOKEN':'token'})
+    mocker.patch.dict(os.environ,{'DATABRICKS_INSECURE':'insecure'})
+
+    config = auth.get_auth_config()
+
+    # Assert
+    assert config != None
+    assert config.host == 'host'
+    assert config.token == 'token'
+    assert config.insecure == 'insecure'

tests/databricks/test_httpretrier.py

@@ -0,0 +1,92 @@
+import pytest
+from common.httpretrier import HTTPRetrier
+import requests
+from requests.exceptions import HTTPError
+from databricks_api import DatabricksAPI
+
+def test__execute__no_exception__returns_value():
+    retrier =  HTTPRetrier()
+    value = 'hello'
+
+    return_value = retrier.execute(_get_value, value)
+
+    assert return_value == value
+
+def test__execute__no_exception_named_args__returns_value():
+    retrier =  HTTPRetrier()
+    value = 'hello'
+
+    return_value = retrier.execute(_get_value, return_value = value)
+
+    assert return_value == value
+
+
+def test__execute__no_exception_named_args_set_first_arg__returns_value():
+    retrier =  HTTPRetrier()
+    value = 'hello'
+
+    return_values = retrier.execute(_get_values, value1 = value)
+
+    assert return_values[0] == value
+    assert return_values[1] is None
+
+
+def test__execute__no_exception_named_args_set_second_arg__returns_value():
+    retrier =  HTTPRetrier()
+    value = 'hello'
+
+    return_values = retrier.execute(_get_values, value2 = value)
+
+    assert return_values[0] is None
+    assert return_values[1] == value
+
+def test__execute__raises_non_http_exception__exception_arises(mocker):
+    retrier =  HTTPRetrier()
+    raiser = ExceptionRaiser(0, ValueError)
+
+    with pytest.raises(ValueError):
+        return_value = retrier.execute(raiser.execute)
+
+def test__execute__raises_500_http_exception__retries_twice_and_raises(mocker):
+    retrier =  HTTPRetrier(2,1)
+
+    db = DatabricksAPI(host='HOST',token='TOKEN')
+    mock_request = mocker.patch.object(db.client.session, 'request')
+    mock_resp = requests.models.Response()
+    mock_resp.status_code = 500
+    mock_request.return_value = mock_resp
+
+    with pytest.raises(HTTPError):
+        return_value = retrier.execute(db.jobs.get_run_output, 1)
+    assert retrier._tries == 2
+
+def test__execute__raises_403_http_exception__no_retries_and_raises(mocker):
+    retrier =  HTTPRetrier(2,1)
+
+    db = DatabricksAPI(host='HOST',token='TOKEN')
+    mock_request = mocker.patch.object(db.client.session, 'request')
+    mock_resp = requests.models.Response()
+    mock_resp.status_code = 403
+    mock_request.return_value = mock_resp
+
+    with pytest.raises(HTTPError):
+        return_value = retrier.execute(db.jobs.get_run_output, 1)
+    assert retrier._tries == 0
+
+def _get_value(return_value):
+    return return_value
+
+def _get_values(value1=None, value2=None):
+    return value1, value2
+
+class ExceptionRaiser(object):
+    def __init__(self, raise_after, exception):
+        self._raise_after = raise_after
+        self._called = 1
+        self._exception = exception
+
+    def execute(self):
+        if self._called > self._raise_after:
+            raise self._exception()
+        self._called = self._called + 1
+        return self._called

tests/databricks/test_utils.py

@@ -0,0 +1,48 @@
+import pytest
+import common.utils as utils
+
+
+def test__recursive_find__2_levels_value__value(mocker):
+    keys = ["a", "b"]
+    test_dict = __get_test_dict()
+    value = utils.recursive_find(test_dict, keys)
+
+    assert value == "c"
+
+
+def test__recursive_find__3_levels_no_value__none(mocker):
+    keys = ["a", "b", "c"]
+    test_dict = __get_test_dict()
+    value = utils.recursive_find(test_dict, keys)
+
+    assert value is None
+
+
+def test__recursive_find__3_levels_value__value(mocker):
+    keys = ["a", "C", "D"]
+    test_dict = __get_test_dict()
+    value = utils.recursive_find(test_dict, keys)
+
+    assert value == "E"
+
+
+def test__recursive_find__3_levels_value__value(mocker):
+    keys = ["a", "C", "D"]
+    test_dict = __get_test_dict()
+    value = utils.recursive_find(test_dict, keys)
+
+    assert value == "E"
+
+
+def test__recursive_find__2_levels_dict__dict(mocker):
+    keys = ["a", "C"]
+    test_dict = __get_test_dict()
+    value = utils.recursive_find(test_dict, keys)
+
+    assert isinstance(value, dict)
+
+
+def __get_test_dict():
+    test_dict = {"a": {"b": "c", "C": {"D": "E"}}, "1": {"2": {"3": "4"}}}
+
+    return test_dict

tests/nutter/test_api.py

@@ -0,0 +1,548 @@
+import pytest
+import os
+import json
+from common.api import Nutter, TestNotebook, NutterStatusEvents
+import common.api as nutter_api
+from common.testresult import TestResults, TestResult
+from common.api import TestNamePatternMatcher
+from common.resultreports import JunitXMLReportWriter
+from common.resultreports import TagsReportWriter
+from common.apiclient import WorkspacePath, DatabricksAPIClient
+from common.statuseventhandler import StatusEventsHandler, EventHandler, StatusEvent
+
+def test__workspacepath__empty_object_response__instance_is_created():
+    objects = {}
+    workspace_path = WorkspacePath.from_api_response(objects)
+
+def test__get_report_writer__junitxmlreportwriter__valid_instance():
+    writer = nutter_api.get_report_writer('JunitXMLReportWriter')
+
+    assert isinstance(writer, JunitXMLReportWriter)
+
+
+def test__get_report_writer__tagsreportwriter__valid_instance():
+    writer = nutter_api.get_report_writer('TagsReportWriter')
+
+    assert isinstance(writer, TagsReportWriter)
+
+
+def test__list_tests__onetest__okay(mocker):
+
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/mynotebook'), ('NOTEBOOK', '/test_mynotebook')])
+
+    nutter.dbclient.list_objects.return_value = workspace_path_1
+
+    tests = nutter.list_tests("/")
+
+    assert len(tests) == 1
+    assert tests[0] == TestNotebook('test_mynotebook', '/test_mynotebook')
+
+
+def test__list_tests__onetest_in_folder__okay(mocker):
+
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/folder/mynotebook'), ('NOTEBOOK', '/folder/test_mynotebook')])
+
+    nutter.dbclient.list_objects.return_value = workspace_path_1
+
+    tests = nutter.list_tests("/folder")
+
+    assert len(tests) == 1
+    assert tests[0] == TestNotebook(
+        'test_mynotebook', '/folder/test_mynotebook')
+
+
+@pytest.mark.skip('No longer needed')
+def test__list_tests__response_without_root_object__okay(mocker):
+
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    objects = """{"objects":[
+    {"object_type":"NOTEBOOK","path":"/mynotebook","language":"PYTHON"},
+    {"object_type":"NOTEBOOK","path":"/test_mynotebook","language":"PYTHON"}]}"""
+
+    nutter.dbclient.list_notebooks.return_value = WorkspacePath(json.loads(objects)[
+        'objects'])
+
+    tests = nutter.list_tests("/")
+
+    assert len(tests) == 1
+    assert tests[0] == TestNotebook('test_mynotebook', '/test_mynotebook')
+
+
+def test__list_tests__onetest_uppercase_name__okay(mocker):
+
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/mynotebook'), ('NOTEBOOK', '/TEST_mynote')])
+
+    nutter.dbclient.list_objects.return_value = workspace_path_1
+
+    tests = nutter.list_tests("/")
+
+    assert len(tests) == 1
+    assert tests == [TestNotebook('TEST_mynote', '/TEST_mynote')]
+
+
+def test__list_tests__nutterstatusevents_testlisting_sequence_is_fired(mocker):
+    event_handler = TestEventHandler()
+    nutter = _get_nutter(mocker, event_handler)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/mynotebook'), ('NOTEBOOK', '/TEST_mynote')])
+
+    nutter.dbclient.list_objects.return_value = workspace_path_1
+
+    tests = nutter.list_tests("/")
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestsListing
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestsListingResults
+    assert status_event.data == 1
+
+def test__list_tests_recursively__1test1dir1test__2_tests(mocker):
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/test_1'), ('DIRECTORY', '/p')])
+    workspace_path_2 = _get_workspacepathobject([('NOTEBOOK', '/p/test_1')])
+
+    nutter.dbclient.list_objects.side_effect = [
+        workspace_path_1, workspace_path_2]
+
+    tests = nutter.list_tests("/", True)
+
+    expected = [TestNotebook('test_1', '/test_1'),
+                TestNotebook('test_1', '/p/test_1')]
+    assert expected == tests
+    assert nutter.dbclient.list_objects.call_count == 2
+
+
+def test__list_tests_recursively__1test1dir2test__3_tests(mocker):
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/test_1'), ('DIRECTORY', '/p')])
+    workspace_path_2 = _get_workspacepathobject(
+        [('NOTEBOOK', '/p/test_1'), ('NOTEBOOK', '/p/test_2')])
+
+    nutter.dbclient.list_objects.side_effect = [
+        workspace_path_1, workspace_path_2]
+
+    tests = nutter.list_tests("/", True)
+    expected = [TestNotebook('test_1', '/test_1'), TestNotebook('test_1',
+                                                                '/p/test_1'), TestNotebook('test_2', '/p/test_2')]
+    assert expected == tests
+    assert nutter.dbclient.list_objects.call_count == 2
+
+
+def test__list_tests_recursively__1test1dir1dir__1_test(mocker):
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/test_1'), ('DIRECTORY', '/p')])
+    workspace_path_2 = _get_workspacepathobject([('DIRECTORY', '/p/c')])
+    workspace_path_3 = _get_workspacepathobject([])
+
+    nutter.dbclient.list_objects.side_effect = [
+        workspace_path_1, workspace_path_2, workspace_path_3]
+
+    tests = nutter.list_tests("/", True)
+
+    expected = [TestNotebook('test_1', '/test_1')]
+    assert expected == tests
+    assert nutter.dbclient.list_objects.call_count == 3
+
+
+def test__list_tests__notest__empty_list(mocker):
+    nutter = _get_nutter(mocker)
+    dbapi_client = _get_client(mocker)
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [
+                                ('NOTEBOOK', '/my'), ('NOTEBOOK', '/my2')])
+
+    results = nutter.list_tests("/")
+
+    assert len(results) == 0
+
+
+
+def test__run_tests__onematch_two_tests___nutterstatusevents_testlisting_scheduling_execution_sequence_is_fired(mocker):
+    event_handler = TestEventHandler()
+    nutter = _get_nutter(mocker, event_handler)
+    test_results = TestResults()
+    test_results.append(TestResult('case',True, 10,[]))
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', test_results.serialize())
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [(
+        'NOTEBOOK', '/test_my'), ('NOTEBOOK', '/test_abc')])
+
+    results = nutter.run_tests("/my*", "cluster")
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestExecutionRequest
+    assert status_event.data == '/my*'
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestsListing
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestsListingResults
+    assert status_event.data == 2
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestsListingFiltered
+    assert status_event.data == 1
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestScheduling
+    assert status_event.data == '/test_my'
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestExecuted
+    assert status_event.data.success
+
+    status_event = event_handler.get_item()
+    assert status_event.event == NutterStatusEvents.TestExecutionResult
+    assert status_event.data #True if success
+
+
+def test__run_tests__onematch__okay(mocker):
+    nutter = _get_nutter(mocker)
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [(
+        'NOTEBOOK', '/test_my'), ('NOTEBOOK', '/my')])
+
+    results = nutter.run_tests("/my*", "cluster")
+
+    assert len(results) == 1
+    result = results[0]
+    assert result.task_result_state == 'TERMINATED'
+
+def test__run_tests_recursively__1test1dir2test__3_tests(mocker):
+    nutter = _get_nutter(mocker)
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+    nutter.dbclient = dbapi_client
+
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('NOTEBOOK', '/test_1'), ('DIRECTORY', '/p')])
+    workspace_path_2 = _get_workspacepathobject(
+        [('NOTEBOOK', '/p/test_1'), ('NOTEBOOK', '/p/test_2')])
+
+    nutter.dbclient.list_objects.side_effect = [
+        workspace_path_1, workspace_path_2]
+
+    tests = nutter.run_tests('/','cluster', 120, 1, True)
+    assert len(tests) == 3
+
+def test__run_tests_recursively__1dir1dir2test__2_tests(mocker):
+    nutter = _get_nutter(mocker)
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+    nutter.dbclient = dbapi_client
+
+    mocker.patch.object(nutter.dbclient, 'list_objects')
+
+    workspace_path_1 = _get_workspacepathobject(
+        [('DIRECTORY', '/p')])
+    workspace_path_2 = _get_workspacepathobject(
+        [('DIRECTORY', '/c')])
+    workspace_path_3 = _get_workspacepathobject(
+        [('NOTEBOOK', '/p/c/test_1'), ('NOTEBOOK', '/p/c/test_2')])
+
+    nutter.dbclient.list_objects.side_effect = [
+        workspace_path_1, workspace_path_2, workspace_path_3]
+
+    tests = nutter.run_tests('/','cluster', 120, 1, True)
+    assert len(tests) == 2
+
+def test__run_tests__onematch_suffix_is_uppercase__okay(mocker):
+    nutter = _get_nutter(mocker)
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [(
+        'NOTEBOOK', '/TEST_my'), ('NOTEBOOK', '/my')])
+
+    results = nutter.run_tests("/my*", "cluster")
+
+    assert len(results) == 1
+    assert results[0].task_result_state == 'TERMINATED'
+
+
+def test__run_tests__nomatch_case_sensitive__okay(mocker):
+    nutter = _get_nutter(mocker)
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [(
+        'NOTEBOOK', '/test_MY'), ('NOTEBOOK', '/my')])
+
+    results = nutter.run_tests("/my*", "cluster")
+
+    assert len(results) == 0
+
+
+def test__run_tests__twomatches_with_pattern__okay(mocker):
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+
+    nutter = _get_nutter(mocker)
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [(
+        'NOTEBOOK', '/test_my'), ('NOTEBOOK', '/test_my2')])
+
+    results = nutter.run_tests("/my*", "cluster")
+
+    assert len(results) == 2
+    assert results[0].task_result_state == 'TERMINATED'
+    assert results[1].task_result_state == 'TERMINATED'
+
+
+def test__run_tests__with_invalid_pattern__valueerror(mocker):
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+    nutter = _get_nutter(mocker)
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [(
+        'NOTEBOOK', '/test_my'), ('NOTEBOOK', '/test_my2')])
+
+    with pytest.raises(ValueError):
+        results = nutter.run_tests("/my/(", "cluster")
+
+
+def test__run_tests__nomatches__okay(mocker):
+
+    submit_response = _get_submit_run_response('SUCCESS', 'TERMINATED', '')
+    dbapi_client = _get_client_for_execute_notebook(mocker, submit_response)
+    nutter = _get_nutter(mocker)
+    nutter.dbclient = dbapi_client
+    _mock_dbclient_list_objects(mocker, dbapi_client, [(
+        'NOTEBOOK', '/test_my'), ('NOTEBOOK', '/test_my2')])
+
+    results = nutter.run_tests("/abc*", "cluster")
+
+    assert len(results) == 0
+
+
+def test__to_testresults__none_output__none(mocker):
+    output = None
+    result = nutter_api.to_testresults(output)
+
+    assert result is None
+
+
+def test__to_testresults__non_pickle_output__none(mocker):
+    output = 'NOT A PICKLE'
+    result = nutter_api.to_testresults(output)
+
+    assert result is None
+
+
+def test__to_testresults__pickle_output__testresult(mocker):
+    output = TestResults().serialize()
+    result = nutter_api.to_testresults(output)
+
+    assert isinstance(result, TestResults)
+
+
+patterns = [
+    (''),
+    ('*'),
+    (None),
+    ('abc'),
+    ('abc*'),
+]
+@pytest.mark.parametrize('pattern', patterns)
+def test__testnamepatternmatcher_ctor_valid_pattern__instance(pattern):
+    pattern_matcher = TestNamePatternMatcher(pattern)
+
+    assert isinstance(pattern_matcher, TestNamePatternMatcher)
+
+
+all_patterns = [
+    (''),
+    ('*'),
+    (None),
+]
+@pytest.mark.parametrize('pattern', all_patterns)
+def test__testnamepatternmatcher_ctor_valid_all_pattern__pattern_is_none(pattern):
+    pattern_matcher = TestNamePatternMatcher(pattern)
+
+    assert isinstance(pattern_matcher, TestNamePatternMatcher)
+    assert pattern_matcher._pattern is None
+
+
+reg_patterns = [
+    ('t?as'),
+    ('tt*'),
+    ('e^6'),
+]
+@pytest.mark.parametrize('pattern', reg_patterns)
+def test__testnamepatternmatcher_ctor_valid_regex_pattern__pattern_is_pattern(pattern):
+    pattern_matcher = TestNamePatternMatcher(pattern)
+
+    assert isinstance(pattern_matcher, TestNamePatternMatcher)
+    assert pattern_matcher._pattern == pattern
+
+
+filter_patterns = [
+    ('', [], 0),
+    ('a', [TestNotebook("test_a", "/test_a")], 1),
+    ('*', [TestNotebook("test_a", "/test_a"), TestNotebook("test_b", "/test_b")], 2),
+    ('b*',[TestNotebook("test_a", "/test_a"), TestNotebook("test_b", "/test_b")], 1),
+    ('b*',[TestNotebook("test_ba", "/test_ba"), TestNotebook("test_b", "/test_b")], 2),
+    ('c*',[TestNotebook("test_a", "/test_a"), TestNotebook("test_b", "/test_b")], 0),
+]
+@pytest.mark.parametrize('pattern, list_results, expected_count', filter_patterns)
+def test__filter_by_pattern__valid_scenarios__result_len_is_expected_count(pattern, list_results, expected_count):
+
+    pattern_matcher = TestNamePatternMatcher(pattern)
+    filtered = pattern_matcher.filter_by_pattern(list_results)
+
+    assert len(filtered) == expected_count
+
+
+invalid_patterns = [
+    ('('),
+    ('--)'),
+]
+@pytest.mark.parametrize('pattern', invalid_patterns)
+def test__testnamepatternmatcher_ctor__invali_pattern__valueerror(pattern):
+
+    with pytest.raises(ValueError):
+        pattern_matcher = TestNamePatternMatcher(pattern)
+
+
+def _get_submit_run_response(result_state, life_cycle_state, result):
+    data_json = """
+                {"notebook_output":
+                {"result": "IHaveReturned", "truncated": false},
+                "metadata":
+                {"execution_duration": 15000,
+                "run_type": "SUBMIT_RUN",
+                "cleanup_duration": 0,
+                "number_in_job": 1,
+                "cluster_instance":
+                {"cluster_id": "0925-141d1222-narcs242",
+                "spark_context_id": "803963628344534476"},
+                "creator_user_name": "abc@microsoft.com",
+                "task": {"notebook_task": {"notebook_path": "/mynotebook"}},
+                "run_id": 7, "start_time": 1569887259173,
+                "job_id": 4,
+                "state": {"result_state": "SUCCESS", "state_message": "",
+                "life_cycle_state": "TERMINATED"}, "setup_duration": 2000,
+                "run_page_url": "https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1",
+                "cluster_spec": {"existing_cluster_id": "0925-141122-narcs242"}, "run_name": "myrun"}}
+                """
+    data_dict = json.loads(data_json)
+    data_dict['notebook_output']['result'] = result
+    data_dict['metadata']['state']['result_state'] = result_state
+    data_dict['metadata']['state']['life_cycle_state'] = life_cycle_state
+
+    return json.dumps(data_dict)
+
+
+def _get_client_for_execute_notebook(mocker, output_data):
+    run_id = {}
+    run_id['run_id'] = 1
+
+    db = _get_client(mocker)
+    mocker.patch.object(db.inner_dbclient.jobs, 'submit_run')
+    db.inner_dbclient.jobs.submit_run.return_value = run_id
+    mocker.patch.object(db.inner_dbclient.jobs, 'get_run_output')
+    db.inner_dbclient.jobs.get_run_output.return_value = json.loads(
+        output_data)
+
+    return db
+
+
+def _get_client(mocker):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': 'myhost'})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': 'mytoken'})
+
+    return DatabricksAPIClient()
+
+
+def _get_nutter(mocker, event_handler = None):
+    mocker.patch.dict(os.environ, {'DATABRICKS_HOST': 'myhost'})
+    mocker.patch.dict(os.environ, {'DATABRICKS_TOKEN': 'mytoken'})
+
+    return Nutter(event_handler)
+
+
+def _mock_dbclient_list_objects(mocker, dbclient, objects):
+    mocker.patch.object(dbclient, 'list_objects')
+
+    workspace_objects = _get_workspacepathobject(objects)
+    dbclient.list_objects.return_value = workspace_objects
+
+
+def _get_workspacepathobject(objects):
+    objects_list = []
+    for object in objects:
+        item = {}
+        item['object_type'] = object[0]
+        item['path'] = object[1]
+        item['language'] = 'PYTHON'
+        objects_list.append(item)
+
+    root_obj = {'objects': objects_list}
+
+    return WorkspacePath.from_api_response(root_obj)
+
+
+class TestEventHandler(EventHandler):
+    def __init__(self):
+        self._queue = None
+        super().__init__()
+
+    def handle(self, queue):
+        self._queue = queue
+
+    def get_item(self):
+        item = self._queue.get()
+        self._queue.task_done()
+        return item
+

tests/nutter/test_apiclientresults.py

@@ -0,0 +1,107 @@
+import pytest
+import json
+from common.api import Nutter, TestNotebook, NutterStatusEvents
+import common.api as nutter_api
+from common.testresult import TestResults, TestResult
+from common.apiclientresults import ExecuteNotebookResult, NotebookOutputResult
+
+def test__is_any_error__not_terminated__true():
+    exec_result = _get_run_test_response('', 'SKIPPED','')
+
+    assert exec_result.is_any_error
+
+
+def test__is_any_error__terminated_not_success__true():
+    exec_result = _get_run_test_response('FAILED', 'TERMINATED','')
+
+    assert exec_result.is_any_error
+
+
+def test__is_any_error__terminated_success_invalid_results__true():
+    exec_result = _get_run_test_response('SUCCESS', 'TERMINATED','')
+
+    assert exec_result.is_any_error
+
+
+def test__is_any_error__terminated_success_valid_results_with_failure__true():
+    test_results = TestResults()
+    test_results.append(TestResult('case',False, 10,[]))
+    exec_result = _get_run_test_response('SUCCESS', 'TERMINATED',test_results.serialize())
+
+    assert exec_result.is_any_error
+
+
+
+def test__is_any_error__terminated_success_valid_results_with_no_failure__false():
+    test_results = TestResults()
+    test_results.append(TestResult('case',True, 10,[]))
+    exec_result = _get_run_test_response('SUCCESS', 'TERMINATED',test_results.serialize())
+
+    assert not exec_result.is_any_error
+
+
+
+def test__is_any_error__terminated_success_2_valid_results_with_no_failure__false():
+    test_results = TestResults()
+    test_results.append(TestResult('case',True, 10,[]))
+    test_results.append(TestResult('case2',True, 10,[]))
+    exec_result = _get_run_test_response('SUCCESS', 'TERMINATED',test_results.serialize())
+
+    assert not exec_result.is_any_error
+
+def test__is_any_error__terminated_success_2_results_1_invalid__true():
+    test_results = TestResults()
+    test_results.append(TestResult('case',True, 10,[]))
+    test_results.append(TestResult('case2',False, 10,[]))
+    exec_result = _get_run_test_response('SUCCESS', 'TERMINATED',test_results.serialize())
+
+    assert exec_result.is_any_error
+
+def test__is_run_from_notebook__result_state_NA__returns_true():
+    # Arrange
+    nbr = NotebookOutputResult('N/A', None, None)
+
+    # Act
+    is_run_from_notebook = nbr.is_run_from_notebook
+
+    #Assert
+    assert True == is_run_from_notebook
+
+def test__is_error__is_run_from_notebook_true__returns_false():
+    # Arrange
+    nbr = NotebookOutputResult('N/A', None, None)
+
+    # Act
+    is_error = nbr.is_error
+
+    #Assert
+    assert False == is_error
+
+def _get_run_test_response(result_state, life_cycle_state, notebook_result):
+    data_json = """
+                {"notebook_output":
+                {"result": "IHaveReturned", "truncated": false},
+                "metadata":
+                {"execution_duration": 15000,
+                "run_type": "SUBMIT_RUN",
+                "cleanup_duration": 0,
+                "number_in_job": 1,
+                "cluster_instance":
+                {"cluster_id": "0925-141d1222-narcs242",
+                "spark_context_id": "803963628344534476"},
+                "creator_user_name": "abc@microsoft.com",
+                "task": {"notebook_task": {"notebook_path": "/test_mynotebook"}},
+                "run_id": 7, "start_time": 1569887259173,
+                "job_id": 4,
+                "state": {"result_state": "SUCCESS", "state_message": "",
+                "life_cycle_state": "TERMINATED"}, "setup_duration": 2000,
+                "run_page_url": "https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1",
+                "cluster_spec": {"existing_cluster_id": "0925-141122-narcs242"}, "run_name": "myrun"}}
+                """
+    data_dict = json.loads(data_json)
+    data_dict['notebook_output']['result'] = notebook_result
+    data_dict['metadata']['state']['result_state'] = result_state
+    data_dict['metadata']['state']['life_cycle_state'] = life_cycle_state
+
+    return ExecuteNotebookResult.from_job_output(data_dict)
+

tests/nutter/test_resultreports.py

@@ -0,0 +1,39 @@
+import pytest
+from common.testresult import TestResults, TestResult
+from common.resultreports import JunitXMLReportWriter
+from common.resultreports import TagsReportWriter
+
+def test_junitxmlreportwriter_add_result__invalid_params__raises_valueerror():
+    writer = JunitXMLReportWriter()
+
+    with pytest.raises(ValueError):
+        writer.add_result(None, None)
+
+
+def test_tagsreportwriter_add_result__invalid_params__raises_valueerror():
+    writer = TagsReportWriter()
+
+    with pytest.raises(ValueError):
+        writer.add_result(None, None)
+
+
+def test_tagsreportwriter_add_result__1_test_result__1_valid_row():
+    writer = TagsReportWriter()
+    test_results = TestResults()
+    test_name = 'case1'
+    duration = 10
+    tags = ['hello', 'hello']
+    test_result = TestResult(test_name, True, duration, tags)
+    test_results.append(test_result)
+    notebook_name = 'test_mynotebook'
+
+    writer.add_result(notebook_name, test_results)
+
+    assert len(writer._rows) == 1
+    row = writer._rows[0]
+
+    assert row.notebook_name == notebook_name
+    assert row.test_name == test_name
+    assert row.passed_str == 'PASSED'
+    assert row.duration == duration
+    assert row.tags == row._to_tag_string(tags)

tests/nutter/test_resultsview.py

@@ -0,0 +1,279 @@
+import json
+import pytest
+from common.resultsview import RunCommandResultsView, TestCaseResultView, ListCommandResultView, ListCommandResultsView
+from common.apiclientresults import ExecuteNotebookResult
+from common.testresult import TestResults, TestResult
+from common.api import TestNotebook
+
+def test__add_exec_result__vaid_instance__isadded(mocker):
+
+    test_results = TestResults().serialize()
+    notebook_results = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results)
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    assert run_results_view.total == 1
+
+
+def test__add_exec_result__vaid_instance_invalid_output__isadded(mocker):
+
+    test_results = "NO PICKLE"
+    notebook_results = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results)
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    assert run_results_view.total == 1
+
+    run_result_view = run_results_view.run_results[0]
+
+    assert len(run_result_view.test_cases_views) == 0
+
+
+def test__add_exec_result__vaid_instance_invalid_output__no_test_case_view(mocker):
+
+    test_results = "NO PICKLE"
+    notebook_results = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results)
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    assert run_results_view.total == 1
+
+
+def test__add_exec_result__vaid_instance__test_case_view(mocker):
+
+    test_results = TestResults()
+    test_case = TestResult("mycase", True, 10, [])
+    test_results.append(test_case)
+
+    notebook_results = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    assert run_results_view.total == 1
+
+    run_result_view = run_results_view.run_results[0]
+
+    assert len(run_result_view.test_cases_views) == 1
+
+    tc_result_view = run_result_view.test_cases_views[0]
+
+    assert tc_result_view.test_case == test_case.test_name
+    assert tc_result_view.passed == test_case.passed
+    assert tc_result_view.execution_time == test_case.execution_time
+
+
+def test__add_exec_result__vaid_instance_two_test_cases__two_test_case_view(mocker):
+
+    test_results = TestResults()
+    test_case = TestResult("mycase", True, 10, [])
+    test_results.append(test_case)
+    test_case = TestResult("mycase2", True, 10, [])
+    test_results.append(test_case)
+
+    notebook_results = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', test_results.serialize())
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    assert run_results_view.total == 1
+
+    run_result_view = run_results_view.run_results[0]
+
+    assert len(run_result_view.test_cases_views) == 2
+
+
+def test__get_view__for_testcase_passed__returns_correct_string(mocker):
+    # Arrange
+    test_case = TestResult("mycase", True, 10, [])
+    test_case_result_view = TestCaseResultView(test_case)
+
+    expected_view = "mycase (10 seconds)\n"
+
+    # Act
+    view = test_case_result_view.get_view()
+
+    # Assert
+    assert expected_view == view
+
+
+def test__get_view__for_testcase_failed__returns_correct_string(mocker):
+    # Arrange
+    stack_trace = "Stack Trace"
+    exception = AssertionError("1 == 2")
+    test_case = TestResult("mycase", False, 5.43, [
+                           'tag1', 'tag2'], exception, stack_trace)
+    test_case_result_view = TestCaseResultView(test_case)
+
+    expected_view = "mycase (5.43 seconds)\n\n" + \
+        stack_trace + "\n\n" + "AssertionError: 1 == 2" + "\n"
+
+    # Act
+    view = test_case_result_view.get_view()
+
+    # Assert
+    assert expected_view == view
+
+
+def test__get_view__for_run_command_result_with_passing_test_case__shows_test_result_under_passing(mocker):
+
+    test_results = TestResults()
+    test_case = TestResult("mycase", True, 10, [])
+    test_results.append(test_case)
+    test_case_result_view = TestCaseResultView(test_case)
+    serialized_results = test_results.serialize()
+
+    notebook_results = __get_ExecuteNotebookResult(
+        'SUCCESS', 'TERMINATED', serialized_results)
+
+    #expected_view = 'Name:            \t/test_mynotebook\nNotebook Exec Result:\tTERMINATED \nTests Cases:\nCase:\tmycase\n\n\tPASSED\n\t\n\t\n\tDuration: 10\n\nCase:\tmycase2\n\n\tPASSED\n\t\n\t\n\tDuration: 10\n\n\n----------------------------------------\n'
+    expected_view = '\nNotebook: /test_mynotebook - Lifecycle State: TERMINATED, Result: SUCCESS\n'
+    expected_view += 'Run Page URL: {}\n'.format(notebook_results.notebook_run_page_url)
+    expected_view += '============================================================\n'
+    expected_view += 'PASSING TESTS\n'
+    expected_view += '------------------------------------------------------------\n'
+    expected_view += test_case_result_view.get_view()
+    expected_view += '\n\n'
+    expected_view += '============================================================\n'
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    view = run_results_view.get_view()
+
+    assert expected_view == view
+
+
+def test__get_view__for_run_command_result_with_failing_test_case__shows_test_result_under_failing(mocker):
+    test_results = TestResults()
+
+    stack_trace = "Stack Trace"
+    exception = AssertionError("1 == 2")
+    test_case = TestResult("mycase", False, 5.43, [
+                           'tag1', 'tag2'], exception, stack_trace)
+    test_case_result_view = TestCaseResultView(test_case)
+    test_results.append(test_case)
+
+    passing_test_case1 = TestResult("mycase1", True, 10, [])
+    test_results.append(passing_test_case1)
+    passing_test_case_result_view1 = TestCaseResultView(passing_test_case1)
+
+    passing_test_case2 = TestResult("mycase2", True, 10, [])
+    test_results.append(passing_test_case2)
+    passing_test_case_result_view2 = TestCaseResultView(passing_test_case2)
+
+    serialized_results = test_results.serialize()
+
+    notebook_results = __get_ExecuteNotebookResult(
+        'FAILURE', 'TERMINATED', serialized_results)
+
+    expected_view = '\nNotebook: /test_mynotebook - Lifecycle State: TERMINATED, Result: FAILURE\n'
+    expected_view += 'Run Page URL: {}\n'.format(notebook_results.notebook_run_page_url)
+    expected_view += '============================================================\n'
+    expected_view += 'FAILING TESTS\n'
+    expected_view += '------------------------------------------------------------\n'
+    expected_view += test_case_result_view.get_view()
+    expected_view += '\n\n'
+    expected_view += 'PASSING TESTS\n'
+    expected_view += '------------------------------------------------------------\n'
+    expected_view += passing_test_case_result_view1.get_view()
+    expected_view += passing_test_case_result_view2.get_view()
+    expected_view += '\n\n'
+    expected_view += '============================================================\n'
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    view = run_results_view.get_view()
+
+    assert expected_view == view
+
+def test__get_view__for_list_command__with_tests_Found__shows_listing(mocker):
+    test_notebook1 = TestNotebook('test_one','/test_one')
+    test_notebook2 = TestNotebook('test_two','/test_two')
+    test_notebooks = [test_notebook1, test_notebook2]
+    list_result_view1 = ListCommandResultView.from_test_notebook(test_notebook1)
+    list_result_view2 = ListCommandResultView.from_test_notebook(test_notebook2)
+
+
+    expected_view = '\nTests Found\n'
+    expected_view += '-------------------------------------------------------\n'
+    expected_view += list_result_view1.get_view()
+    expected_view += list_result_view2.get_view()
+    expected_view += '-------------------------------------------------------\n'
+
+    list_results_view = ListCommandResultsView(test_notebooks)
+
+    view = list_results_view.get_view()
+
+    assert view == expected_view
+
+
+
+def test__get_view__for_run_command_result_with_one_passing_one_failing__shows_failing_then_passing(mocker):
+
+    stack_trace = "Stack Trace"
+    exception = AssertionError("1 == 2")
+    test_case = TestResult("mycase", False, 5.43, [
+                           'tag1', 'tag2'], exception, stack_trace)
+    test_case_result_view = TestCaseResultView(test_case)
+
+    test_results = TestResults()
+    test_results.append(test_case)
+    serialized_results = test_results.serialize()
+
+    notebook_results = __get_ExecuteNotebookResult(
+        'FAILURE', 'TERMINATED', serialized_results)
+
+    expected_view = '\nNotebook: /test_mynotebook - Lifecycle State: TERMINATED, Result: FAILURE\n'
+    expected_view += 'Run Page URL: {}\n'.format(notebook_results.notebook_run_page_url)
+    expected_view += '============================================================\n'
+    expected_view += 'FAILING TESTS\n'
+    expected_view += '------------------------------------------------------------\n'
+    expected_view += test_case_result_view.get_view()
+    expected_view += '\n\n'
+    expected_view += '============================================================\n'
+
+    run_results_view = RunCommandResultsView()
+    run_results_view.add_exec_result(notebook_results)
+
+    view = run_results_view.get_view()
+
+    assert expected_view == view
+
+
+def __get_ExecuteNotebookResult(result_state, life_cycle_state, notebook_result):
+    data_json = """
+                {"notebook_output":
+                {"result": "IHaveReturned", "truncated": false},
+                "metadata":
+                {"execution_duration": 15000,
+                "run_type": "SUBMIT_RUN",
+                "cleanup_duration": 0,
+                "number_in_job": 1,
+                "cluster_instance":
+                {"cluster_id": "0925-141d1222-narcs242",
+                "spark_context_id": "803963628344534476"},
+                "creator_user_name": "abc@microsoft.com",
+                "task": {"notebook_task": {"notebook_path": "/test_mynotebook"}},
+                "run_id": 7, "start_time": 1569887259173,
+                "job_id": 4,
+                "state": {"result_state": "SUCCESS", "state_message": "",
+                "life_cycle_state": "TERMINATED"}, "setup_duration": 2000,
+                "run_page_url": "https://westus2.azuredatabricks.net/?o=14702dasda6094293890#job/4/run/1",
+                "cluster_spec": {"existing_cluster_id": "0925-141122-narcs242"}, "run_name": "myrun"}}
+                """
+    data_dict = json.loads(data_json)
+    data_dict['notebook_output']['result'] = notebook_result
+    data_dict['metadata']['state']['result_state'] = result_state
+    data_dict['metadata']['state']['life_cycle_state'] = life_cycle_state
+
+    return ExecuteNotebookResult.from_job_output(data_dict)

tests/nutter/test_scheduler.py

@@ -0,0 +1,87 @@
+import pytest
+import common.scheduler as scheduler
+import time
+import datetime
+
+
+def test__run_and_wait__1_function_1_worker_exception__result_is_none_and_exception():
+    func_scheduler = scheduler.get_scheduler(1)
+    func_scheduler.add_function(__raise_it, Exception)
+    results = func_scheduler.run_and_wait()
+    assert len(results) == 1
+    assert results[0].func_result is None
+    assert isinstance(results[0].exception, Exception)
+
+
+params = [
+    (1, 1, 'this'),
+    (1, 2, 'this'),
+    (2, 2, 'this'),
+    (2, 10, 'this'),
+    (2, 2, {'this': 'this'}),
+    (2, 2, ('this', 'that')),
+]
+@pytest.mark.parametrize('num_of_funcs, num_of_workers, func_return_value', params)
+def test__run_and_wait__X_functions_X_workers_x_value__results_are_okay(num_of_funcs, num_of_workers, func_return_value):
+    func_scheduler = scheduler.get_scheduler(num_of_workers)
+
+    for i in range(0, num_of_funcs):
+        func_scheduler.add_function(__get_back, func_return_value)
+
+    results = func_scheduler.run_and_wait()
+    assert len(results) == num_of_funcs
+
+    for result in results:
+        assert result.func_result == func_return_value
+
+
+def test__run_and_wait__3_function_1_worker__in_sequence():
+    func_scheduler = scheduler.get_scheduler(1)
+    value1 = 'this1'
+    func_scheduler.add_function(__get_back, value1)
+    value2 = 'this2'
+    func_scheduler.add_function(__get_back, value2)
+    value3 = 'this3'
+    func_scheduler.add_function(__get_back, value3)
+    results = func_scheduler.run_and_wait()
+    assert len(results) == 3
+    assert results[0].func_result == value1
+    assert results[1].func_result == value2
+    assert results[2].func_result == value3
+
+
+def test__run_and_wait__2_functions_1_worker_500ms_delay__sequential_duration():
+    func_scheduler = scheduler.get_scheduler(1)
+    wait_time = .500
+    func_scheduler.add_function(__wait, wait_time)
+    func_scheduler.add_function(__wait, wait_time)
+    start = time.time()
+    results = func_scheduler.run_and_wait()
+    end = time.time()
+    delay = int(end - start)
+    assert delay >= 2 * wait_time
+
+
+def test__run_and_wait__3_functions_3_worker_500ms_delay__less_than_sequential_duration():
+    func_scheduler = scheduler.get_scheduler(1)
+    wait_time = .500
+    func_scheduler.add_function(__wait, wait_time)
+    func_scheduler.add_function(__wait, wait_time)
+    func_scheduler.add_function(__wait, wait_time)
+    start = time.time()
+    results = func_scheduler.run_and_wait()
+    end = time.time()
+    delay = int(end - start)
+    assert delay < 3 * wait_time
+
+
+def __get_back(this):
+    return this
+
+
+def __raise_it(exception):
+    raise exception
+
+
+def __wait(time_to_wait):
+    time.sleep(time_to_wait)

tests/nutter/test_statuseventhandler.py

@@ -0,0 +1,46 @@
+import pytest
+import enum
+from common.statuseventhandler import StatusEventsHandler, EventHandler, StatusEvent
+
+def test__add_event_and_wait__1_event__handler_receives_it():
+    test_handler = TestEventHandler()
+    status_handler = StatusEventsHandler(test_handler)
+
+    status_handler.add_event(TestStatusEvent.AnEvent, 'added')
+    item = test_handler.get_item()
+    status_handler.wait()
+
+    assert item.event == TestStatusEvent.AnEvent
+    assert item.data == 'added'
+
+def test__add_event_and_wait__2_event2__handler_receives_them():
+    test_handler = TestEventHandler()
+    status_handler = StatusEventsHandler(test_handler)
+
+    status_handler.add_event(TestStatusEvent.AnEvent, 'added')
+    status_handler.add_event(TestStatusEvent.AnEvent, 'added')
+    item = test_handler.get_item()
+    item2 = test_handler.get_item()
+    status_handler.wait()
+
+    assert item.event == TestStatusEvent.AnEvent
+    assert item.data == 'added'
+
+    assert item2.event == TestStatusEvent.AnEvent
+    assert item2.data == 'added'
+
+class TestEventHandler(EventHandler):
+    def __init__(self):
+        self._queue = None
+        super().__init__()
+
+    def handle(self, queue):
+        self._queue = queue
+
+    def get_item(self):
+        item = self._queue.get()
+        self._queue.task_done()
+        return item
+
+class TestStatusEvent(enum.Enum):
+    AnEvent = 1

tests/nutter/test_testexecresults.py

@@ -0,0 +1,83 @@
+import pytest
+from common.testexecresults import TestExecResults
+from common.testresult import TestResults, TestResult
+
+def test__ctor__test_results_not_correct_type__raises_type_error():
+    with pytest.raises(TypeError):
+        test_exec_result = TestExecResults("invalidtype")
+
+def test__to_string__valid_test_results__creates_view_from_test_results_and_returns(mocker):
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("test1", True, 10, []))
+    test_results.append(TestResult("test2", True, 10, []))
+
+    test_exec_result = TestExecResults(test_results)
+
+    mocker.patch.object(test_exec_result, 'get_ExecuteNotebookResult')
+    notebook_result = TestExecResults(test_results).get_ExecuteNotebookResult("", test_results)
+    test_exec_result.get_ExecuteNotebookResult.return_value = notebook_result
+
+    mocker.patch.object(test_exec_result.runcommand_results_view, 'add_exec_result')
+    mocker.patch.object(test_exec_result.runcommand_results_view, 'get_view')
+    test_exec_result.runcommand_results_view.get_view.return_value = "expectedview"
+    
+    # Act
+    view = test_exec_result.to_string()
+
+    # Assert
+    test_exec_result.get_ExecuteNotebookResult.assert_called_once_with("", test_results)
+    test_exec_result.runcommand_results_view.add_exec_result.assert_called_once_with(notebook_result)
+    test_exec_result.runcommand_results_view.get_view.assert_called_once_with()
+    assert view == "expectedview"
+
+def test__to_string__valid_test_results_run_from_notebook__creates_view_from_test_results_and_returns(mocker):
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("test1", True, 10, []))
+    test_results.append(TestResult("test2", True, 10, []))
+
+    test_exec_result = TestExecResults(test_results)
+
+    # Act
+    view = test_exec_result.to_string()
+
+    # Assert
+    assert "PASSING TESTS" in view
+    assert "test1" in view
+    assert "test2" in view
+
+def test__exit__valid_test_results__serializes_test_results_and_passes_to_dbutils_exit(mocker):
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("test1", True, 10, []))
+    test_results.append(TestResult("test2", True, 10, []))
+
+    test_exec_result = TestExecResults(test_results)
+
+    mocker.patch.object(test_results, 'serialize')
+    serialized_data = "serializeddata"
+    test_results.serialize.return_value = serialized_data
+
+    dbutils_stub = DbUtilsStub()
+
+    # Act
+    test_exec_result.exit(dbutils_stub)
+
+    # Assert
+    test_results.serialize.assert_called_with()
+    assert True == dbutils_stub.notebook.exit_called
+    assert serialized_data == dbutils_stub.notebook.data_passed
+
+class DbUtilsStub:
+    def __init__(self):
+        self.notebook = NotebookStub()
+
+class NotebookStub():
+    def __init__(self):
+        self.exit_called = False
+        self.data_passed = ""
+
+    def exit(self, data):
+        self.exit_called = True
+        self.data_passed = data

tests/nutter/test_testresult.py

@@ -0,0 +1,136 @@
+import pytest
+import json
+from common.testresult import TestResults, TestResult
+import pickle
+import base64
+
+def test__testresults_append__type_not_testresult__throws_error():
+    # Arrange
+    test_results = TestResults()
+
+    # Act/Assert
+    with pytest.raises(TypeError):
+        test_results.append("Test")
+
+def test__testresults_append__type_testresult__appends_testresult():
+    # Arrange
+    test_results = TestResults()
+
+    # Act
+    test_results.append(TestResult("Test Name", True, 1, []))
+    
+    # Assert
+    assert len(test_results.results) == 1
+
+def test__eq__test_results_not_equal__are_not_equal():
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("Test NameX", True, 1, []))
+    test_results.append(TestResult("Test Name1", True, 1, [], ValueError("Error")))
+
+    test_results1 = TestResults()
+    test_results1.append(TestResult("Test Name", True, 1, []))
+    test_results1.append(TestResult("Test Name1", True, 1, [], ValueError("Error")))
+
+    # Act / Assert
+    are_not_equal = test_results != test_results1
+    assert are_not_equal == True
+
+def test__deserialize__no_constraints__is_serializable_and_deserializable():
+    # Arrange
+    test_results = TestResults()
+
+    test_results.append(TestResult("Test Name", True, 1, []))
+    test_results.append(TestResult("Test Name1", True, 1, [], ValueError("Error")))
+
+    serialized_data = test_results.serialize()
+
+    deserialized_data = TestResults().deserialize(serialized_data)
+
+    assert test_results == deserialized_data
+
+def test__deserialize__empty_pickle_data__throws_exception():
+    # Arrange
+    test_results = TestResults()
+
+    invalid_pickle = ""
+
+    # Act / Assert
+    with pytest.raises(Exception):
+        test_results.deserialize(invalid_pickle)
+
+def test__deserialize__invalid_pickle_data__throws_Exception():
+    # Arrange
+    test_results = TestResults()
+
+    invalid_pickle = "test"
+
+    # Act / Assert
+    with pytest.raises(Exception):
+        test_results.deserialize(invalid_pickle)
+
+
+def test__eq__test_results_equal_but_not_same_ref__are_equal():
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("Test Name", True, 1, []))
+    test_results.append(TestResult("Test Name1", True, 1, [], ValueError("Error")))
+
+    test_results1 = TestResults()
+    test_results1.append(TestResult("Test Name", True, 1, []))
+    test_results1.append(TestResult("Test Name1", True, 1, [], ValueError("Error")))
+
+    # Act / Assert
+    assert test_results == test_results1
+
+def test__num_tests__5_test_cases__is_5():
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("Test Name", True, 1, []))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+
+    # Act / Assert
+    assert 5 == test_results.test_cases
+
+def test__num_failures__5_test_cases_4_failures__is_4():
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("Test Name", True, 1, []))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 1, [], ValueError("Error")))
+
+    # Act / Assert
+    assert 4 == test_results.num_failures
+
+def test__total_execution_time__5_test_cases__is_sum_of_execution_times():
+    # Arrange
+    test_results = TestResults()
+    test_results.append(TestResult("Test Name", True, 1.12, []))
+    test_results.append(TestResult("Test Name1", False, 1.0005, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 10.000034, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 7.66, [], ValueError("Error")))
+    test_results.append(TestResult("Test Name1", False, 13.21, [], ValueError("Error")))
+
+    # Act / Assert
+    assert 32.990534 == test_results.total_execution_time
+
+def test__serialize__result_data__is_base64_str():
+    test_results = TestResults()
+    serialized_data = test_results.serialize()
+    serialized_bin_data = base64.encodebytes(pickle.dumps(test_results))
+    
+    assert serialized_data == str(serialized_bin_data, "utf-8")
+
+
+def test__deserialize__data_is_base64_str__can_deserialize():
+    test_results = TestResults()
+    serialized_bin_data =  pickle.dumps(test_results)
+    serialized_str = str(base64.encodebytes(serialized_bin_data), "utf-8")
+    test_results_from_data = TestResults().deserialize(serialized_str)
+
+    assert test_results == test_results_from_data

tests/runtime/test_fixtureloader.py

@@ -0,0 +1,176 @@
+import pytest
+from runtime.fixtureloader import FixtureLoader
+from tests.runtime.testnutterfixturebuilder import TestNutterFixtureBuilder
+
+def test__get_fixture_loader__returns_fixtureloader():
+    # Arrange / Act
+    loader = FixtureLoader()
+
+    # Assert
+    assert isinstance(loader, FixtureLoader)
+
+def test__load_fixture__none_passed_raises__valueerror():
+    # Arrange 
+    loader = FixtureLoader()
+
+    # Act
+    with pytest.raises(ValueError):
+        loader.load_fixture(None)
+ 
+def test__load_fixture__one_assertion_method__adds_one_testclass_to_dictionary_with_assert_set():
+    # Arrange
+    test_name = "fred"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name) \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert len(loaded_fixture) == 1
+    __assert_test_case_from_dict(loaded_fixture, test_name, True, True, False, True)
+
+def test__load_fixture__one_assertion_method_one_additional_method__adds_one_testclass_to_dictionary_with_assert_set():
+    # Arrange
+    test_name = "fred"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name) \
+        .with_test(test_name) \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert len(loaded_fixture) == 1
+    __assert_test_case_from_dict(loaded_fixture, test_name, True, True, False, True)
+
+def test__load_fixture__one_assertion_one_run_method__adds_one_testclass_to_dictionary_with_assert_and_run_set():
+    # Arrange
+    test_name = "fred"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name) \
+        .with_run(test_name) \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert len(loaded_fixture) == 1
+    __assert_test_case_from_dict(loaded_fixture, test_name, True, False, False, True)
+
+def test__load_fixture__before_all__no_test_case_set_because_method_exists_on_fixture():
+    # Arrange
+    test_name = "fred"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name) \
+        .with_run(test_name) \
+        .with_before_all() \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert "before_all" not in loaded_fixture
+    assert "all" not in loaded_fixture
+
+def test__load_fixture__after_all__no_test_case_set_because_method_exists_on_fixture():
+    # Arrange
+    test_name = "fred"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name) \
+        .with_run(test_name) \
+        .with_after_all() \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert "after_all" not in loaded_fixture
+    assert "all" not in loaded_fixture
+
+def test__load_fixture__two_assertion_one_run_method__adds_two_testclass_to_dictionary():
+    # Arrange
+    test_name_1 = "fred"
+    test_name_2 = "hank"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name_1) \
+        .with_assertion(test_name_2) \
+        .with_run(test_name_1) \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert len(loaded_fixture) == 2
+    __assert_test_case_from_dict(loaded_fixture, test_name_1, True, False, False, True)
+    __assert_test_case_from_dict(loaded_fixture, test_name_2, True, True, False, True)
+
+def test__load_fixture__three_with_all_methods__adds_three_testclass_to_dictionary():
+    # Arrange
+    test_name_1 = "fred"
+    test_name_2 = "hank"
+    test_name_3 = "will"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_before(test_name_1) \
+        .with_before(test_name_2) \
+        .with_before(test_name_3) \
+        .with_run(test_name_1) \
+        .with_run(test_name_2) \
+        .with_run(test_name_3) \
+        .with_assertion(test_name_1) \
+        .with_assertion(test_name_2) \
+        .with_assertion(test_name_3) \
+        .with_after(test_name_1) \
+        .with_after(test_name_2) \
+        .with_after(test_name_3) \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert len(loaded_fixture) == 3
+    __assert_test_case_from_dict(loaded_fixture, test_name_1, False, False, False, False)
+    __assert_test_case_from_dict(loaded_fixture, test_name_2, False, False, False, False)
+    __assert_test_case_from_dict(loaded_fixture, test_name_3, False, False, False, False)
+
+
+def __assert_test_case_from_dict(test_case_dict, expected_name, before_none, run_none, assertion_none, after_none):
+    assert expected_name in test_case_dict
+
+    test_case = test_case_dict[expected_name]
+    assert (test_case.before is None) == before_none
+    assert (test_case.run is None) == run_none
+    assert (test_case.assertion is None) == assertion_none
+    assert (test_case.after is None) == after_none
+
+  
+
+

tests/runtime/test_nutterfixture.py

@@ -0,0 +1,335 @@
+import pytest
+from runtime.nutterfixture import NutterFixture, tag, InvalidTestFixtureException
+from runtime.testcase import TestCase
+from common.testresult  import TestResult, TestResults
+from tests.runtime.testnutterfixturebuilder import TestNutterFixtureBuilder
+from common.apiclientresults import ExecuteNotebookResult
+import sys
+
+def test__ctor__creates_fixture_loader():
+    # Arrange / Act
+    fix = SimpleTestFixture()
+
+    # Assert
+    assert fix.data_loader is not None
+
+def test__execute_tests__calls_load_fixture_on_fixture_loader(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+
+    # Act
+    fix.execute_tests()
+
+    # Assert
+    fix.data_loader.load_fixture.assert_called_once_with(fix)
+
+def test__execute_tests__data_loader_returns_none__throws_invalidfixtureexception(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = None
+
+    # Act / Assert
+    with pytest.raises(InvalidTestFixtureException):
+        fix.execute_tests()
+
+def test__execute_tests__data_loader_returns_empty_dictionary__returns_empty_results(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = {}
+
+    # Act
+    test_exec_results = fix.execute_tests()
+
+    # Assert
+    assert len(test_exec_results.test_results.results) == 0
+
+def test__execute_tests__before_all_set_and_data_loader_returns_empty_dictionary__does_not_call_before_all(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = {}
+    fix.before_all = lambda self: 1 == 1
+
+    mocker.patch.object(fix, 'before_all')
+
+    # Act
+    test_results = fix.execute_tests()
+
+    # Assert
+    fix.before_all.assert_not_called()
+
+def test__execute_tests__before_all_none_and_data_loader_returns_empty_dictionary__does_not_call_before_all(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = {}
+    fix.before_all = None
+
+    mocker.patch.object(fix, 'before_all')
+
+    # Act
+    test_results = fix.execute_tests()
+
+    # Assert
+    fix.before_all.assert_not_called()
+
+def test__execute_tests__before_all_set_and_data_loader_returns_dictionary_with_testcases__calls_before_all(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+
+    tc = __get_test_case("TestName", fix.run_test, fix.assertion_test)
+    fix.before_all = lambda self: 1 == 1
+    mocker.patch.object(fix, 'before_all')
+
+    test_case_dict = {
+        "test": tc
+        }
+
+    fix.data_loader.load_fixture.return_value = test_case_dict
+
+    # Act 
+    fix.execute_tests()
+
+    # Assert
+    fix.before_all.assert_called_once_with()
+
+def test__execute_tests__after_all_set_and_data_loader_returns_empty_dictionary__does_not_call_after_all(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = {}
+    fix.after_all = lambda self: 1 == 1
+
+    mocker.patch.object(fix, 'after_all')
+
+    # Act
+    test_results = fix.execute_tests()
+
+    # Assert
+    fix.after_all.assert_not_called()
+
+def test__execute_tests__after_all_none_and_data_loader_returns_empty_dictionary__does_not_call_after_all(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = {}
+    fix.after_all = None
+
+    mocker.patch.object(fix, 'after_all')
+
+    # Act
+    test_results = fix.execute_tests()
+
+    # Assert
+    fix.after_all.assert_not_called()
+
+def test__execute_tests__after_all_set_and_data_loader_returns_dictionary_with_testcases__calls_after_all(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+
+    tc = __get_test_case("TestName", fix.run_test, fix.assertion_test)
+    fix.after_all = lambda self: 1 == 1
+    mocker.patch.object(fix, 'after_all')
+
+    test_case_dict = {
+        "test": tc
+        }
+
+    fix.data_loader.load_fixture.return_value = test_case_dict
+
+    # Act 
+    fix.execute_tests()
+
+    # Assert
+    fix.after_all.assert_called_once_with()
+
+def test__execute_tests__data_loader_returns_dictionary_with_testcases__iterates_over_dictionary_and_calls_execute(mocker):
+    # Arrange 
+    fix = SimpleTestFixture()
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+
+    tc = __get_test_case("TestName", fix.run_test, fix.assertion_test)
+    mocker.patch.object(tc, 'execute_test')
+    tc.execute_test.return_value = TestResult("TestName", True, 1, [])
+    tc1 = __get_test_case("TestName", fix.run_test, fix.assertion_test)
+    mocker.patch.object(tc1, 'execute_test')
+    tc1.execute_test.return_value = TestResult("TestName", True, 1, [])
+
+    test_case_dict = {
+        "test": tc,
+        "test1": tc1
+        }
+
+    fix.data_loader.load_fixture.return_value = test_case_dict
+
+    # Act 
+    fix.execute_tests()
+
+    # Assert
+    tc.execute_test.assert_called_once_with()
+    tc1.execute_test.assert_called_once_with()
+
+def test__execute_tests__returns_test_result__calls_append_on_testresults(mocker):
+    # Arrange
+    fix = SimpleTestFixture()
+    mocker.patch.object(fix.test_results, 'append')
+
+    tc = __get_test_case("TestName", lambda: 1 == 1, lambda: 1 == 1)
+
+    test_case_dict = {
+        "test": tc
+        }
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = test_case_dict
+
+    # Act 
+    result = fix.execute_tests()
+
+    # Assert 
+    fix.test_results.append.assert_called_once_with(mocker.ANY)
+
+def test__execute_tests__two_test_cases__returns_test_results_with_2_test_results(mocker):
+    # Arrange
+    fix = SimpleTestFixture()
+
+    tc = __get_test_case("TestName", lambda: 1 == 1, lambda: 1 == 1)
+    tc1 = __get_test_case("TestName1", lambda: 1 == 1, lambda: 1 == 1)
+
+    test_case_dict = {
+        "TestName": tc,
+        "TestName1": tc1
+        }
+
+    mocker.patch.object(fix.data_loader, 'load_fixture')
+    fix.data_loader.load_fixture.return_value = test_case_dict
+
+    # Act 
+    result = fix.execute_tests()
+
+    # Assert 
+    assert len(result.test_results.results) == 2
+
+
+def test__run_test_method__has_list_tag_decorator__list_set_on_method():
+    # Arrange
+    class Wrapper(NutterFixture):
+        tag_list = ["tag1", "tag2"]
+        @tag(tag_list)
+        def run_test(self):
+            lambda: 1 == 1
+
+    test_name = "test"
+    tag_list = ["tag1", "tag2"]
+
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name) \
+        .with_run(test_name, Wrapper.run_test) \
+        .build()
+
+    # Act / Assert
+    assert tag_list == test_fixture.run_test.tag
+
+def test__run_test_method__has_str_tag_decorator__str_set_on_method():
+    # Arrange
+    class Wrapper(NutterFixture):
+        tag_str = "mytag"
+        @tag(tag_str)
+        def run_test(self):
+            lambda: 1 == 1
+
+    test_name = "test"
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name) \
+        .with_run(test_name, Wrapper.run_test) \
+        .build()
+
+    # Act / Assert
+    assert "mytag" == test_fixture.run_test.tag
+ 
+def test__run_test_method__has_tag_decorator_not_list__raises_value_error():
+    # Arrange
+    with pytest.raises(ValueError):
+        class Wrapper(NutterFixture):
+            tag_invalid = {}
+            @tag(tag_invalid)
+            def run_test(self):
+                lambda: 1 == 1
+
+def test__run_test_method__has_tag_decorator_not_listhas_invalid_tag_decorator_none__raises_value_error():
+    # Arrange
+    with pytest.raises(ValueError):
+        class Wrapper(NutterFixture):
+            tag_invalid = None
+            @tag(tag_invalid)
+            def run_test(self):
+                lambda: 1 == 1
+
+def test__non_run_test_method__valid_tag_on_non_run_method__raises_value_error():
+    # Arrange
+    with pytest.raises(ValueError):
+        class Wrapper(NutterFixture):
+            tag_valid = "mytag"
+            @tag(tag_valid)
+            def assertion_test(self):
+                lambda: 1 == 1
+
+def __get_test_case(name, setrun, setassert):
+    tc = TestCase(name)
+    tc.set_run(setrun)
+    tc.set_assertion(setassert)
+
+    return tc
+
+def test__run_test_method__has_invalid_tag_decorator_not_list_or_str_using_class_not_builder__raises_value_error():
+    # Arrange
+    simple_test_fixture = SimpleTestFixture()
+
+    # Act / Assert
+    with pytest.raises(ValueError):
+        simple_test_fixture.run_test_with_invalid_decorator()
+
+def test__run_test_method__has_valid_tag_decorator_in_class__tag_set_on_method():
+    # Arrange
+    simple_test_fixture = SimpleTestFixture()
+
+    # Act / Assert 
+    assert "mytag" == simple_test_fixture.run_test_with_valid_decorator.tag
+
+class SimpleTestFixture(NutterFixture):
+
+    def before_test(self):
+        pass
+
+    def run_test(self):
+        pass
+
+    def assertion_test(self):
+        assert 1 == 1
+
+    def after_test(self):
+        pass
+
+    @tag("mytag")
+    def run_test_with_valid_decorator(self):
+        pass
+
+    @tag
+    def run_test_with_invalid_decorator(self):
+        pass
+

tests/runtime/test_nutterfixure_fullroundtriptests.py

@@ -0,0 +1,164 @@
+import sys
+import pytest
+from runtime.nutterfixture import NutterFixture, tag
+from common.testresult  import TestResult
+from tests.runtime.testnutterfixturebuilder import TestNutterFixtureBuilder
+
+
+def test__execute_tests__two_valid_cases__returns_test_results_with_2_passed_test_results():
+    # Arrange
+    test_name_1 = "fred"
+    test_name_2 = "hank"
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_before(test_name_1) \
+        .with_before(test_name_2) \
+        .with_run(test_name_1) \
+        .with_run(test_name_2) \
+        .with_assertion(test_name_1) \
+        .with_assertion(test_name_2) \
+        .with_after(test_name_1) \
+        .with_after(test_name_2) \
+        .build()
+
+    expected_result1 = TestResult(test_name_1, True, 1, [])
+    expected_result2 = TestResult(test_name_2, True, 1, [])
+
+    # Act 
+    result = test_fixture().execute_tests().test_results
+
+    # Assert 
+    assert len(result.results) == 2
+    assert __item_in_list_equalto(result.results, expected_result1)
+    assert __item_in_list_equalto(result.results, expected_result2)
+
+def test__execute_tests__one_valid_one_invalid__returns_correct_test_results():
+    # Arrange
+    test_name_1 = "shouldpass"
+    test_name_2 = "shouldfail"
+    fail_func = AssertionHelper().assertion_fails
+
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_before(test_name_1) \
+        .with_before(test_name_2) \
+        .with_run(test_name_1) \
+        .with_run(test_name_2) \
+        .with_assertion(test_name_1) \
+        .with_assertion(test_name_2, fail_func) \
+        .with_after(test_name_1) \
+        .with_after(test_name_2) \
+        .build()
+
+    expected_result1 = TestResult(test_name_1, True, 1, [])
+    expected_result2 = TestResult(test_name_2, False, 1, [], AssertionError("assert 1 == 2"))
+
+    # Act 
+    result = test_fixture().execute_tests().test_results
+
+    # Assert 
+    assert len(result.results) == 2
+    assert __item_in_list_equalto(result.results, expected_result1)
+    assert __item_in_list_equalto(result.results, expected_result2)
+
+def test__execute_tests__one_run_throws__returns_one_failed_testresult():
+    # Arrange
+    test_name_1 = "shouldthrow"
+    fail_func = AssertionHelper().function_throws
+
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_before(test_name_1) \
+        .with_run(test_name_1, fail_func) \
+        .with_assertion(test_name_1) \
+        .with_after(test_name_1) \
+        .build()
+
+    expected_result1 = TestResult(test_name_1, False, 1, [], ValueError())
+ 
+    # Act 
+    result = test_fixture().execute_tests().test_results
+
+    # Assert 
+    assert len(result.results) == 1
+    assert __item_in_list_equalto(result.results, expected_result1)
+
+def test__execute_tests__one_has_tags_one_does_not__returns_tags_in_testresult():
+    # Arrange
+    class Wrapper(NutterFixture):
+        tag_list = ["taga", "tagb"]
+        @tag(tag_list)
+        def run_test_name(self):
+            lambda: 1 == 1
+
+    test_name_1 = "test_name"
+    test_name_2 = "test_name2"
+
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name(test_name_1) \
+        .with_run(test_name_1, Wrapper.run_test_name) \
+        .with_assertion(test_name_1) \
+        .with_after(test_name_1) \
+        .with_name(test_name_2) \
+        .with_run(test_name_2) \
+        .with_assertion(test_name_2) \
+        .with_after(test_name_2) \
+        .build()
+
+    # Act 
+    result = test_fixture().execute_tests().test_results
+
+    # Assert 
+    assert len(result.results) == 2
+    for res in result.results:
+        if res.test_name == test_name_1:
+            assert ("taga" in res.tags) == True
+            assert ("tagb" in res.tags) == True
+        if res.test_name == test_name_2:
+            assert len(res.tags) == 0
+
+def test__execute_tests__one_test_case_with_all_methods__all_methods_called(mocker):
+    # Arrange
+    test_name_1 = "test"
+    
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_before_all() \
+        .with_before(test_name_1) \
+        .with_run(test_name_1) \
+        .with_assertion(test_name_1) \
+        .with_after(test_name_1) \
+        .with_after_all() \
+        .build()
+
+    mocker.patch.object(test_fixture, 'before_all')
+    mocker.patch.object(test_fixture, 'before_test')
+    mocker.patch.object(test_fixture, 'run_test')
+    mocker.patch.object(test_fixture, 'assertion_test')
+    mocker.patch.object(test_fixture, 'after_test')
+    mocker.patch.object(test_fixture, 'after_all')
+
+    # Act 
+    result = test_fixture().execute_tests()
+
+    # Assert
+    test_fixture.before_all.assert_called_once_with()
+    test_fixture.before_test.assert_called_once_with()
+    test_fixture.run_test.assert_called_once_with()
+    test_fixture.assertion_test.assert_called_once_with()
+    test_fixture.after_test.assert_called_once_with()
+    test_fixture.after_all.assert_called_once_with()
+
+def __item_in_list_equalto(list, expected_item):
+    for item in list:
+        if (item == expected_item):
+            return True
+    
+    return False
+    
+class AssertionHelper():
+    def assertion_fails(self):
+        assert 1 == 2
+    def function_throws(self):
+        raise ValueError()
+

tests/runtime/test_testcase.py

@@ -0,0 +1,418 @@
+import os
+import pytest
+import time
+from common.testresult  import TestResult
+from runtime.nutterfixture import tag
+from runtime.testcase import TestCase, NoTestCasesFoundError
+
+def test__isvalid_rundoesntexist_returnsfalse():
+    # Arrange
+    tc = TestCase("Test Name")
+    fixture = TestFixture()
+    
+    tc.set_assertion(fixture.assertion_test)
+
+    # Act
+    isvalid = tc.is_valid()
+
+    # Assert
+    assert False == isvalid 
+
+def test__isvalid_assertiondoesntexist_returnsfalse():
+    # Arrange
+    tc = TestCase("Test Name")
+    fixture = TestFixture()
+    
+    tc.set_run(fixture.run_test)
+
+    # Act
+    isvalid = tc.is_valid()
+
+    # Assert
+    assert False == isvalid
+
+def test__isvalid_runandassertionexist_returnstrue():
+    # Arrange
+    tc = TestCase("Test Name")
+    fixture = TestFixture()
+    
+    tc.set_assertion(fixture.assertion_test)
+    tc.set_run(fixture.run_test)
+
+    # Act
+    isvalid = tc.is_valid()
+
+    # Assert
+    assert True == isvalid
+
+def test__getinvalidmessage_rundoesntexist_returnsrunerrormessage():
+    # Arrange
+    tc = TestCase("Test Name")
+    fixture = TestFixture()
+    
+    tc.set_assertion(fixture.assertion_test)
+
+    expected_message = tc.ERROR_MESSAGE_RUN_MISSING 
+
+    # Act
+    invalid_message = tc.get_invalid_message()
+
+    # Assert
+    assert expected_message == invalid_message
+
+def test__getinvalidmessage_assertiondoesntexist_returnsassertionerrormessage():
+    # Arrange
+    tc = TestCase("Test Name")
+    fixture = TestFixture()
+    
+    tc.set_run(fixture.run_test)
+
+    expected_message = tc.ERROR_MESSAGE_ASSERTION_MISSING 
+
+    # Act
+    invalid_message = tc.get_invalid_message()
+
+    # Assert
+    assert expected_message == invalid_message
+
+def test__getinvalidmessage_runandassertiondontexist_returnsrunandassertionerrormessage():
+    # Arrange
+    tc = TestCase("Test Name")
+    fixture = TestFixture()
+    
+    # Act
+    invalid_message = tc.get_invalid_message()
+
+    # Assert
+    assertion_message_exists = tc.ERROR_MESSAGE_ASSERTION_MISSING in invalid_message
+    run_message_exists = tc.ERROR_MESSAGE_RUN_MISSING in invalid_message
+
+    assert assertion_message_exists == True
+    assert run_message_exists == True
+
+def test__set_run__function_passed__sets_run_function():
+    # Arrange
+    tc = TestCase("Test Name")
+    fixture = TestFixture()
+    
+    # Act
+    tc.set_run(fixture.run_test)
+
+    # Assert
+    assert tc.run == fixture.run_test
+
+def test__set_assertion__function_passed__sets_assertion_function():
+    # Arrange
+    tc = TestCase("Test Name")
+    func = lambda: 1 == 1 
+    
+    # Act
+    tc.set_assertion(func)
+
+    # Assert
+    assert tc.assertion == func
+
+def test__set_before__function_passed__sets_before_function():
+    # Arrange
+    tc = TestCase("Test Name")
+    func = lambda: 1 == 1 
+    
+    # Act
+    tc.set_before(func)
+
+    # Assert
+    assert tc.before == func
+
+def test__set_after__function_passed__sets_after_function():
+    # Arrange
+    tc = TestCase("Test Name")
+    func = lambda: 1 == 1 
+    
+    # Act
+    tc.set_after(func)
+
+    # Assert
+    assert tc.after == func
+
+def test__execute_test__before_set__calls_before(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+
+    tc.set_before(lambda: 1 == 1)
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+    mocker.patch.object(tc, 'before')
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    tc.before.assert_called_once_with()
+
+def test__execute_test__before_not_set__does_not_call_before(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+    mocker.patch.object(tc, 'before')
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    tc.before.assert_not_called()
+
+def test__execute_test__after_set__calls_after(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+
+    tc.set_after(lambda: 1 == 1)
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+    mocker.patch.object(tc, 'after')
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    tc.after.assert_called_once_with()
+
+def test__execute_test__after_not_set__does_not_call_after(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+    mocker.patch.object(tc, 'after')
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    tc.after.assert_not_called()
+
+def test__execute_test__method_in_assert_doesnt_throw__returns_pass_testresult(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+    fixture = TestFixture()
+
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result == TestResult("TestName", True, None, [], None)
+
+def test__execute_test__is_valid_equals_false__returns_fail_testresult():
+    # Arrange
+    tc = TestCase("TestName")
+    no_test_cases_error = NoTestCasesFoundError('Both a run and an assertion are required for every test')
+
+    ## (Note - no set_assertion - so invalid)
+    tc.set_run(lambda: 1 == 1)
+    
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result == TestResult("TestName", False, 1, [], no_test_cases_error)
+
+def test__execute_test__method_in_assert_throws__returns_fail_testresult():
+    # Arrange
+    tc = TestCase("TestName")
+    assertion_error = AssertionError('bad assert')
+
+    lambda_that_throws = lambda: (_ for _ in ()).throw(assertion_error)
+
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda_that_throws)
+
+    # Act
+    test_result = tc.execute_test()
+    # Assert
+    assert test_result == TestResult("TestName", False, 1, [], assertion_error)
+
+
+def test__execute_test__method_in_run_throws__returns_fail_testresult():
+    # Arrange
+    tc = TestCase("TestName")
+    not_implemented_exception = NotImplementedError("Whatever was not implemented")
+
+    lambda_that_throws = lambda: (_ for _ in ()).throw(not_implemented_exception)
+
+    tc.set_run(lambda_that_throws)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result == TestResult("TestName", False, 1, [], not_implemented_exception)
+
+def test__execute_test__method_in_before_throws__returns_fail_testresult():
+    # Arrange
+    tc = TestCase("TestName")
+    not_implemented_exception = NotImplementedError("Whatever was not implemented")
+
+    lambda_that_throws = lambda: (_ for _ in ()).throw(not_implemented_exception)
+
+    tc.set_before(lambda_that_throws)
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result == TestResult("TestName", False, 1, [], not_implemented_exception)
+
+def test__execute_test__method_in_after_throws__returns_fail_testresult():
+    # Arrange
+    tc = TestCase("TestName")
+    not_implemented_exception = NotImplementedError("Whatever was not implemented")
+
+    lambda_that_throws = lambda: (_ for _ in ()).throw(not_implemented_exception)
+
+    tc.set_after(lambda_that_throws)
+    tc.set_before(lambda: 1 == 1)
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result == TestResult("TestName", False, 1, [], not_implemented_exception)
+
+def test__execute_test__method_throws__returns_stacktrace_in_testresult():
+    # Arrange
+    tc = TestCase("TestName")
+    not_implemented_exception = NotImplementedError("Whatever was not implemented")
+
+    lambda_that_throws = lambda: (_ for _ in ()).throw(not_implemented_exception)
+
+    tc.set_run(lambda_that_throws)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result.stack_trace
+
+def test__execute_test__no_constraints__sets_execution_time():
+    # Arrange
+    tc = TestCase("TestName")
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+    
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result.execution_time > 0
+
+def test__run_method__no_tags__tags_list_empty(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert len(tc.tags) == 0
+
+def test__run_method__string_tag__tags_list_contains_string(mocker):
+    # Arrange
+    strtag = "testtag"
+    @tag(strtag)
+    def run_TestName():
+        lambda: 1 == 1
+
+    tc = TestCase("TestName")
+    tc.set_run(run_TestName)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert [strtag] == tc.tags
+
+def test__run_method__list_tag__tags_list_contains_list(mocker):
+    # Arrange
+    tag_list = ["taga", "tagb"]
+    @tag(tag_list)
+    def run_TestName():
+        lambda: 1 == 1
+
+    tc = TestCase("TestName")
+    tc.set_run(run_TestName)
+    tc.set_assertion(lambda: 1 == 1)
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert tc.tags == tag_list
+
+def test__execute__run_has_tag__test_results_returns_tags():
+    # Arrange
+    tag_list = ["taga", "tagb"]
+    @tag(tag_list)
+    def run_TestName():
+        lambda: 1 == 1
+
+    tc = TestCase("TestName")
+    tc.set_run(run_TestName)
+    tc.set_assertion(lambda: 1 == 1)
+    
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result.tags == tag_list
+
+def test__execute__run_has_tag_and_execute_fails__test_results_returns_tags():
+    # Arrange
+    tag_list = ["taga", "tagb"]
+    @tag(tag_list)
+    def run_TestName():
+        pass
+
+    def assertion_TestName():
+        assert 1 == 2
+    
+    tc = TestCase("TestName")
+    tc.set_run(run_TestName)
+    tc.set_assertion(assertion_TestName)
+    
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    assert test_result.tags == tag_list
+
+class TestFixture():
+
+    # def before_all(self):
+    #     return True
+    def before_test(self):
+        return True
+
+    def run_test(self):
+        return True
+    def throw(self):
+        raise AssertionError("Method not implemented")
+    def assertion_test(self):
+        return True
+
+    def after_test(self):
+        return True
+    # def after_all(self):
+    #     return True

tests/runtime/testnutterfixturebuilder.py

@@ -0,0 +1,47 @@
+from runtime.nutterfixture import NutterFixture
+
+class TestNutterFixtureBuilder():
+    def __init__(self):
+        self.attributes = {}
+        self.class_name = "ImplementingClass"
+
+    def with_name(self, class_name):
+        self.class_name = class_name
+        return self
+
+    def with_before_all(self, func = lambda self: 1 == 1):
+        self.attributes.update({"before_all" : func })
+        return self
+
+    def with_before(self, test_name, func = lambda self: 1 == 1):
+        full_test_name = "before_" + test_name
+        self.attributes.update({full_test_name : func})
+        return self
+
+    def with_assertion(self, test_name, func = lambda self: 1 == 1):
+        full_test_name = "assertion_" + test_name
+        self.attributes.update({full_test_name : func})
+        return self
+
+    def with_run(self, test_name, func = lambda self: 1 == 1):
+        full_test_name = "run_" + test_name
+        self.attributes.update({full_test_name : func})
+        return self
+
+    def with_after(self, test_name, func = lambda self: 1 == 1):
+        full_test_name = "after_" + test_name
+        self.attributes.update({full_test_name : func})
+        return self
+
+    def with_after_all(self, func = lambda self: 1 == 1):
+        self.attributes.update({"after_all" : func })
+        return self
+        
+    def with_test(self, test_name, func = lambda self: 1 == 1):
+        full_test_name = test_name
+        self.attributes.update({full_test_name : func})
+        return self
+
+    def build(self):
+        new_class = type(self.class_name, (NutterFixture,), self.attributes)
+        return new_class