diff --git a/src/pytest_quilla/__init__.py b/src/pytest_quilla/__init__.py index 6e60fb9..e0a949d 100644 --- a/src/pytest_quilla/__init__.py +++ b/src/pytest_quilla/__init__.py @@ -1,3 +1,5 @@ +import uuid + import pytest from _pytest.config import Config from _pytest.config.argparsing import Parser @@ -37,4 +39,7 @@ def pytest_load_initial_conftests(early_config: Config, parser: Parser): def pytest_collect_file(parent: pytest.Session, path): - return collect_file(parent, path, parent.config.getini('quilla-prefix')) + return collect_file(parent, path, parent.config.getini('quilla-prefix'), run_id) + + +run_id = str(uuid.uuid4()) diff --git a/src/pytest_quilla/pytest_classes.py b/src/pytest_quilla/pytest_classes.py index 528a48c..fcd5f58 100644 --- a/src/pytest_quilla/pytest_classes.py +++ b/src/pytest_quilla/pytest_classes.py @@ -10,7 +10,7 @@ from quilla import ( from quilla.reports.report_summary import ReportSummary -def collect_file(parent: pytest.Session, path: LocalPath, prefix: str): +def collect_file(parent: pytest.Session, path: LocalPath, prefix: str, run_id: str): ''' Collects files if their path ends with .json and starts with the prefix @@ -18,6 +18,7 @@ def collect_file(parent: pytest.Session, path: LocalPath, prefix: str): parent: The session object performing the collection path: The path to the file that might be collected prefix: The prefix for files that should be collected + run_id: The run ID of the quilla tests Returns: A quilla file object if the path matches, None otherwise @@ -26,10 +27,14 @@ def collect_file(parent: pytest.Session, path: LocalPath, prefix: str): # TODO: change "path" to be "fspath" when pytest 6.3 is released: # https://docs.pytest.org/en/latest/_modules/_pytest/hookspec.html#pytest_collect_file if path.ext == '.json' and path.basename.startswith(prefix): - return QuillaFile.from_parent(parent, fspath=path) + return QuillaFile.from_parent(parent, fspath=path, run_id=run_id) class QuillaFile(pytest.File): + def __init__(self, *args, run_id: str = '', **kwargs) -> None: + super().__init__(*args, **kwargs) + self.quilla_run_id = run_id + def collect(self): ''' Loads the JSON test data from the path and creates the test instance @@ -38,13 +43,19 @@ class QuillaFile(pytest.File): A quilla item configured from the JSON data ''' test_data = self.fspath.open().read() - yield QuillaItem.from_parent(self, name=self.fspath.purebasename, test_data=test_data) + yield QuillaItem.from_parent( + self, + name=self.fspath.purebasename, + test_data=test_data, + run_id=self.quilla_run_id + ) class QuillaItem(pytest.Item): - def __init__(self, name: str, parent: QuillaFile, test_data: str): + def __init__(self, name: str, parent: QuillaFile, test_data: str, run_id: str): super(QuillaItem, self).__init__(name, parent) self.test_data = test_data + self.quilla_run_id = run_id json_data = json.loads(test_data) markers = json_data.get('markers', []) for marker in markers: @@ -59,6 +70,12 @@ class QuillaItem(pytest.Item): [*self.config.getoption('--quilla-opts').split(), ''], str(self.config.rootpath) ) + if not ( + '-i' in self.config.getoption('--quilla-opts') or + '--run-id' in self.config.getoption('--quilla-opts') + ): + ctx.run_id = self.quilla_run_id + ctx.json = self.test_data results = execute(ctx) self.results = results diff --git a/src/quilla/__init__.py b/src/quilla/__init__.py index 9726f6e..d6ef418 100644 --- a/src/quilla/__init__.py +++ b/src/quilla/__init__.py @@ -30,6 +30,7 @@ def make_parser() -> argparse.ArgumentParser: # pragma: no cover ''' parser = argparse.ArgumentParser( prog='quilla', + usage='%(prog)s [options] [-f] JSON', description=''' Program to provide a report of UI validations given a json representation of the validations or given the filename containing a json document describing @@ -37,6 +38,12 @@ def make_parser() -> argparse.ArgumentParser: # pragma: no cover ''', ) + parser.add_argument( + '--version', + action='store_true', + help='Prints the version of the software and quits' + ) + parser.add_argument( '-f', '--file', @@ -48,25 +55,32 @@ def make_parser() -> argparse.ArgumentParser: # pragma: no cover 'json', help='The json file name or raw json string', ) - parser.add_argument( - '--debug', - action='store_true', - help='Enable debug mode', + + config_group = parser.add_argument_group(title='Configuration options') + config_group.add_argument( + '-i', + '--run-id', + action='store', + metavar='run_id', + default=None, + help='A run ID for quilla, if manually passed in.' + 'Used to set many quilla tests to have the same run ID' ) - parser.add_argument( + config_group.add_argument( + '-d', + '--definitions', + action='append', + metavar='file', + help='A file with definitions for the \'Definitions\' context object' + ) + config_group.add_argument( '--driver-dir', dest='drivers_path', action='store', default='.', help='The directory where browser drivers are stored', ) - parser.add_argument( - '-P', - '--pretty', - action='store_true', - help='Set this flag to have the output be pretty-printed' - ) - parser.add_argument( + config_group.add_argument( '--no-sandbox', dest='no_sandbox', action='store_true', @@ -75,14 +89,28 @@ def make_parser() -> argparse.ArgumentParser: # pragma: no cover Useful for running in docker containers' ''' ) - parser.add_argument( - '-d', - '--definitions', - action='append', - metavar='file', - help='A file with definitions for the \'Definitions\' context object' + + output_group = parser.add_argument_group(title='Output Options') + output_group.add_argument( + '-P', + '--pretty', + action='store_true', + help='Set this flag to have the output be pretty-printed' ) - parser.add_argument( + output_group.add_argument( + '--indent', + type=int, + default=4, + help='How much space each indent level should have when pretty-printing the report' + ) + + debug_group = parser.add_argument_group(title='Debug Options') + debug_group.add_argument( + '--debug', + action='store_true', + help='Enable debug mode', + ) + debug_group.add_argument( '-v', '--verbose', action='count', @@ -90,11 +118,6 @@ def make_parser() -> argparse.ArgumentParser: # pragma: no cover 'Log outputs are directed to stderr by default.', default=0 ) - parser.add_argument( - '--version', - action='store_true', - help='Prints the version of the software and quits' - ) return parser @@ -257,6 +280,8 @@ def setup_context(args: List[str], plugin_root: str = '.') -> Context: parsed_args.no_sandbox, parsed_args.definitions, logger=logger, + run_id=parsed_args.run_id, + indent=parsed_args.indent, ) logger.info('Running "quilla_configure" hook') @@ -279,8 +304,6 @@ def run(): ctx.logger.debug('Finished generating reports') out = reports.to_dict() - if ctx._context_data['Outputs']: - out['Outputs'] = ctx._context_data['Outputs'] if ctx.pretty: print(json.dumps( diff --git a/src/quilla/ctx.py b/src/quilla/ctx.py index 943f61a..5a60cac 100644 --- a/src/quilla/ctx.py +++ b/src/quilla/ctx.py @@ -14,6 +14,7 @@ from logging import ( NullHandler, ) import json +import uuid from pluggy import PluginManager import pydeepmerge as pdm @@ -39,6 +40,7 @@ class Context(DriverHolder): is_file: Whether a file was originally passed in or if the raw json was passed in no_sandbox: Whether to pass the '--no-sandbox' arg to Chrome and Edge logger: An optional configured logger instance. + run_id: A string that uniquely identifies the run of Quilla. Attributes: @@ -54,6 +56,8 @@ class Context(DriverHolder): no_sandbox: Whether to pass the '--no-sandbox' arg to Chrome and Edge logger: A logger instance. If None was passed in for the 'logger' argument, will create one with the default logger. + run_id: A string that uniquely identifies the run of Quilla. + pretty_print_indent: How many spaces to use for indentation when pretty-printing the output ''' default_context: Optional['Context'] = None _drivers_path: str @@ -65,7 +69,6 @@ class Context(DriverHolder): r'([a-zA-Z][a-zA-Z0-9_]+)(\.[a-zA-Z_][a-zA-Z0-9_]+)+' ) _output_browser: str = 'Firefox' - pretty_print_indent: int = 4 def __init__( self, @@ -77,7 +80,9 @@ class Context(DriverHolder): is_file: bool = False, no_sandbox: bool = False, definitions: List[str] = [], - logger: Optional[Logger] = None + logger: Optional[Logger] = None, + run_id: Optional[str] = None, + indent: int = 4, ): super().__init__() self.pm = plugin_manager @@ -87,6 +92,7 @@ class Context(DriverHolder): self.json = json_data self.is_file = is_file self.no_sandbox = no_sandbox + self.pretty_print_indent = indent path = Path(drivers_path) if logger is None: @@ -95,10 +101,22 @@ class Context(DriverHolder): else: self.logger = logger + if run_id is None: + self.run_id = str(uuid.uuid4()) # Generate a random UUID + else: + self.run_id = run_id + self.drivers_path = str(path.resolve()) self._context_data: Dict[str, dict] = {'Validation': {}, 'Outputs': {}, 'Definitions': {}} self._load_definition_files(definitions) + @property + def outputs(self) -> dict: + ''' + A dictionary of all outputs created by the steps for the current Quilla test + ''' + return self._context_data['Outputs'] + @property def is_debug(self) -> bool: ''' @@ -329,7 +347,9 @@ def get_default_context( no_sandbox: bool = False, definitions: List[str] = [], recreate_context: bool = False, - logger: Optional[Logger] = None + logger: Optional[Logger] = None, + run_id: Optional[str] = None, + indent: int = 4, ) -> Context: ''' Gets the default context, creating a new one if necessary. @@ -350,6 +370,7 @@ def get_default_context( recreate_context: Whether a new context object should be created or not logger: An optional logger instance. If None, one will be created with the NullHandler. + run_id: A string that uniquely identifies the run of Quilla. Returns Application context shared for the entire application @@ -368,5 +389,7 @@ def get_default_context( no_sandbox, definitions, logger, + run_id, + indent, ) return Context.default_context diff --git a/src/quilla/reports/base_report.py b/src/quilla/reports/base_report.py index d6ce83c..ea21cdb 100644 --- a/src/quilla/reports/base_report.py +++ b/src/quilla/reports/base_report.py @@ -1,6 +1,5 @@ import json from abc import ( - abstractclassmethod, abstractmethod, ) from typing import Dict @@ -29,7 +28,8 @@ class BaseReport(EnumResolver): self.msg: str = msg self.report_type: ReportType = report_type - @abstractclassmethod + @classmethod + @abstractmethod def from_dict(cls, report: Dict[str, Dict[str, str]]) -> 'BaseReport': ''' Converts a dictionary report into a valid Report object diff --git a/src/quilla/reports/report_summary.py b/src/quilla/reports/report_summary.py index 8daa51e..e818228 100644 --- a/src/quilla/reports/report_summary.py +++ b/src/quilla/reports/report_summary.py @@ -10,6 +10,7 @@ from quilla.common.enums import ReportType from quilla.reports.base_report import BaseReport from quilla.reports.validation_report import ValidationReport from quilla.reports.step_failure_report import StepFailureReport +from quilla.reports.visual_parity_report import VisualParityReport class ReportSummary: @@ -17,9 +18,13 @@ class ReportSummary: A class to describe a series of report objects, as well as manipulating them for test purposes. Args: + run_id: A string that uniquely identifies the run + outputs: The outputs generated by various steps reports: A list of reports to produce a summary of Attributes: + run_id: A string that uniquely identifies the run + outputs: The outputs generated by various steps reports: A list of reports used to produce a summary successes: The number of reports that are described as successful fails: The numer of reports that are not described as successful @@ -30,9 +35,12 @@ class ReportSummary: selector: Dict[str, Type[BaseReport]] = { 'validationReport': ValidationReport, 'stepFailureReport': StepFailureReport, + 'visualParityReport': VisualParityReport, } - def __init__(self, reports: List[BaseReport] = []): + def __init__(self, run_id: str, outputs: dict, reports: List[BaseReport] = []): + self.run_id = run_id + self.outputs = outputs self.reports = reports self.successes = 0 self.fails = 0 @@ -54,7 +62,9 @@ class ReportSummary: 'reports': [ report.to_dict() for report in self.reports ] - } + }, + 'outputs': self.outputs, + 'run_id': self.run_id, } def to_json(self) -> str: @@ -65,12 +75,14 @@ class ReportSummary: return json.dumps(self.to_dict()) @classmethod - def from_dict(cls, summary_dict): + def from_dict(cls, summary_dict: dict): ''' Loads a ReportSummary object that is represented as a dictionary. It does not trust the metadata that is in the report, and will regenerate the metadata itself. ''' reports = summary_dict['reportSummary']['reports'] + run_id = summary_dict.get('run_id', '') + outputs = summary_dict.get('outputs', {}) obj_reports = [] for report in reports: # Each report has a report tag as the root of the json document @@ -78,7 +90,7 @@ class ReportSummary: report_object = cls.selector[report_type] obj_reports.append(report_object.from_dict(report)) obj_reports = [ValidationReport.from_dict(report) for report in reports] - return ReportSummary(obj_reports) + return ReportSummary(run_id, outputs, obj_reports) @classmethod def from_json(cls, summary_json): @@ -110,6 +122,7 @@ class ReportSummary: ''' def __init__(self, summary: 'ReportSummary'): self._summary = summary + self._run_id = summary.run_id def _filter(self, condition: Callable[[BaseReport], bool]) -> 'ReportSummary': ''' @@ -119,7 +132,7 @@ class ReportSummary: reports = self._summary.reports.copy() filtered_reports = filter(condition, reports) - return ReportSummary(list(filtered_reports)) + return ReportSummary(self._run_id, {}, list(filtered_reports)) def state(self, state: str) -> 'ReportSummary': ''' diff --git a/src/quilla/reports/visual_parity_report.py b/src/quilla/reports/visual_parity_report.py index d8170d7..39dc604 100644 --- a/src/quilla/reports/visual_parity_report.py +++ b/src/quilla/reports/visual_parity_report.py @@ -1,4 +1,7 @@ - +from typing import ( + Dict, + cast +) from quilla.common.enums import ( XPathValidationStates, ValidationTypes @@ -75,3 +78,24 @@ class VisualParityReport(ValidationReport): return { 'visualParityReport': report_data } + + @classmethod + def from_dict(cls, report) -> 'VisualParityReport': + params: Dict[str, str] = report['visualParityReport'] + msg = params.get('msg', '') + baseline_id = params['baselineId'] + baseline_uri = params.get('baselineImageUri', '') + treatment_uri = params.get('treatmentImageUri', '') + delta_uri = params.get('deltaImageUri', '') + success = cast(bool, params['passed']) + + return VisualParityReport( + target=params['target'], + browser_name=params['targetBrowser'], + success=success, + msg=msg, + baseline_id=baseline_id, + baseline_image_uri=baseline_uri, + treatment_image_uri=treatment_uri, + delta_image_uri=delta_uri, + ) diff --git a/src/quilla/ui_validation.py b/src/quilla/ui_validation.py index 84ee063..dbccb09 100644 --- a/src/quilla/ui_validation.py +++ b/src/quilla/ui_validation.py @@ -160,4 +160,4 @@ class QuillaTest(EnumResolver): for browser in self.browsers: validation_reports.extend(browser.validate()) - return ReportSummary(validation_reports) + return ReportSummary(self.ctx.run_id, self.ctx.outputs, validation_reports) diff --git a/tests/conftest.py b/tests/conftest.py index 37f7310..1966725 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ from logging import ( Logger, NullHandler, ) +import uuid import pytest from _pytest.config import PytestPluginManager @@ -43,6 +44,9 @@ def driver(): return mock_driver +run_id = str(uuid.uuid4()) + + def pytest_addoption(parser, pluginmanager: PytestPluginManager): pluginmanager.set_blocked('quilla') parser.addoption( @@ -54,4 +58,4 @@ def pytest_addoption(parser, pluginmanager: PytestPluginManager): def pytest_collect_file(parent, path): - return collect_file(parent, path, 'test') + return collect_file(parent, path, 'test', run_id)