From 9cd8a44b46d4110758fcb9cc035684c6cee27e83 Mon Sep 17 00:00:00 2001 From: Natalia Maximo Date: Thu, 15 Jul 2021 14:54:05 -0400 Subject: [PATCH] new: added LocalStorage plugin for VisualParity (#42) --- images/baselines/TrialCompleteBanner.png | Bin 0 -> 1897 bytes setup.cfg | 2 +- src/quilla/__init__.py | 17 +- src/quilla/common/utils.py | 4 +- src/quilla/ctx.py | 12 +- src/quilla/hookspecs/__init__.py | 22 +- src/quilla/hookspecs/_hookspec.py | 3 + src/quilla/hookspecs/configuration.py | 2 +- src/quilla/hookspecs/reports.py | 2 +- src/quilla/hookspecs/steps.py | 19 +- .../{plugins.py => plugins/__init__.py} | 30 ++- src/quilla/plugins/base_storage.py | 202 ++++++++++++++++++ src/quilla/plugins/local_storage.py | 129 +++++++++++ src/quilla/reports/visual_parity_report.py | 6 +- src/quilla/steps/base_steps.py | 2 +- src/quilla/steps/validations/xpath.py | 58 ++++- .../integration/test_quilla_plugins_work.json | 21 -- tests/integration/test_visual_parity.json | 79 +++++++ 18 files changed, 566 insertions(+), 44 deletions(-) create mode 100644 images/baselines/TrialCompleteBanner.png create mode 100644 src/quilla/hookspecs/_hookspec.py rename src/quilla/{plugins.py => plugins/__init__.py} (84%) create mode 100644 src/quilla/plugins/base_storage.py create mode 100644 src/quilla/plugins/local_storage.py delete mode 100644 tests/integration/test_quilla_plugins_work.json create mode 100644 tests/integration/test_visual_parity.json diff --git a/images/baselines/TrialCompleteBanner.png b/images/baselines/TrialCompleteBanner.png new file mode 100644 index 0000000000000000000000000000000000000000..0fe42b25ba10ee74b2788799c367d9d1f79ee389 GIT binary patch literal 1897 zcmb7FX;@Ne8>T)Rit=*O($NUZJ<){9jFB|IOhXZK!vUAbT}vHQTne;t#>~AbaZS*LQxr*K?iop67Y)`?>G;O}m1#SB7Xo zKp>DZ=CZ9T2&9AsV$Ho@0V`a&?L7ztMqq4h+~XgYFL)92gq>Q0lQAcxCDhD`^~}=EYp5F(MRie< zoU66`ZTs^x?Ew3@@vP25k#D^Gg99lxXueRez1sh&uaE5Q2~-E3_#oVK@R#s|<3+?S zMR@nZ{}X(;y5~VgcsMcl=W>B%EOxU%O*_ei|IIiGMef)rG-lS`3(TCVWo1jhQzY-? zThwg5vmtO_tWKqBbJ0&(TzdsXxtD{PP@rSMz z@_DGHFomB_Y2wg&4XG}Oec3rn5}Nc8x0q6F!IOcb-xlt9@F2MV>z%QHKpWNv*?Kk2 z&%N`I6JB0N*ltE&H~X|zPmYWb@8wl*Rae)p%$iw5gm`+;U3c%&pI}2A9VPS2?VqfY zTo9q0c;S9Ma1P94x$}0bR7&|{gFN}?(x5piby`fE`nGmbO3Yes`#dri}pVAo%6yfu8G%ZowTe)!%s_prEwbN94#_7|6niV~Tg{_w7jCeFM zg5c(=!~f=Rq)0t_z=uWyK=4o5rLh}+8g?geUowiB;O_2g^S@>albg>3Hu%1mS)1u! z3T5I?By2_zxJDGQLzdEy*T%v$E9faH#Fp5G^`sC3r)E;ry9Hs*SnG{KyoSaSy~@#i zdfx4pXMVj3EcaFKN2}Mny0t}$3 z22$fOQ>Uc1gdq{kG;w=iiq#?CsDO@n$lsL#{1CVcLu_^d4;^VkEHd%;JGlqo=W&N5 z&%{5M6Hx*?paKl!!U;J3f>Ix0Z3VsOlN=rW?VBT>%L zxR`kN?(r(I_$K;n^+BHSWAbhw{}YMin)IjSSJvNJnggT)y`)C=VJSAroElt4rcUFLn0O|GSsnaqoZ8(-#PRBJ+iqkdSDmf5oy$yp#iz^ z2((F(Z|RfDxwlo0jZIBJdTZ*c%do$W7;ECVDd{gkf=f`{Bvs;PRfqw{?SMf?;Tt?u zx8Px03u9~#Kqqkt3eD*8Pu8cvw9P^ry@fW)*NQl3}fQHzf!{Cd^_-_mTlINJ8PApTdt3y ztuOPu@1Gtbxi*Fu+a=pjV&dSh*g&Uu{CR&oW%aKA^>bimEUpjsj3Q;PpU;2nE9oB_ zADWfHZSSy{lGsSIZg!%~4NM5o;7KArK{R)l}exdn)j!EQ-l1kcUYx7T5Gvn!rw z9RW}PwrV(e67O)VZ2OUpS-qm4+j36678W`-Qr8qrXNS@AHe@@bOlzZ5)v zb3;TESMchTE4p1}Mb3;-=F%AaxVeV`+)*@5b+e|##7og~;_ra3kyqj9AyQ+gG#0=x z^xJBTER3%{*-8^hOIht!ha)B`l^D*;XD#?-Mmv6Ec)SVln^D)iTUpr!u-1_g>y_9t zz!~+CVXil0+3M9bOh0)dfVIm$XSJ<6mEIavr5f(FdFfJEXt>xErRe{K@L{~lyL^K~ zfcS=A&`cCBk6G argparse.ArgumentParser: # pragma: no cover action='store', metavar='run_id', default=None, - help='A run ID for quilla, if manually passed in.' - 'Used to set many quilla tests to have the same run ID' + help='A run ID for quilla, if manually passed in. ' + 'Used to set many quilla tests to have the same run ID ' + 'If no run ID is manually passed in, it will be auto-generated' + ) + config_group.add_argument( + '-u', + '--update-baseline', + dest='update_baseline', + action='store_true', + help='Used to update the baseline images for VisualParity. ' + 'Different plugins define different behaviour, so this is not ' + 'necessarily a lossless operation.' ) config_group.add_argument( '-d', @@ -282,10 +292,11 @@ def setup_context(args: List[str], plugin_root: str = '.') -> Context: logger=logger, run_id=parsed_args.run_id, indent=parsed_args.indent, + update_baseline=parsed_args.update_baseline ) logger.info('Running "quilla_configure" hook') - pm.hook.quilla_configure(ctx=ctx, args=args) + pm.hook.quilla_configure(ctx=ctx, args=parsed_args) return ctx diff --git a/src/quilla/common/utils.py b/src/quilla/common/utils.py index 9bbff8a..379cd8a 100644 --- a/src/quilla/common/utils.py +++ b/src/quilla/common/utils.py @@ -81,7 +81,7 @@ class EnumResolver: if ctx is not None: resolved_plugin_value = ctx.pm.hook.quilla_resolve_enum_from_name(name=name, enum=enum) - if len(resolved_plugin_value) > 0: - return resolved_plugin_value[0] + if resolved_plugin_value is not None: + return resolved_plugin_value raise EnumValueNotFoundException(name, enum) diff --git a/src/quilla/ctx.py b/src/quilla/ctx.py index 5a60cac..56f60d0 100644 --- a/src/quilla/ctx.py +++ b/src/quilla/ctx.py @@ -41,6 +41,7 @@ class Context(DriverHolder): no_sandbox: Whether to pass the '--no-sandbox' arg to Chrome and Edge logger: An optional configured logger instance. run_id: A string that uniquely identifies the run of Quilla. + update_baseline: Whether the VisualParity baselines should be updated or not Attributes: @@ -58,6 +59,7 @@ class Context(DriverHolder): one with the default logger. run_id: A string that uniquely identifies the run of Quilla. pretty_print_indent: How many spaces to use for indentation when pretty-printing the output + update_baseline: Whether the VisualParity baselines should be updated or not ''' default_context: Optional['Context'] = None _drivers_path: str @@ -83,6 +85,7 @@ class Context(DriverHolder): logger: Optional[Logger] = None, run_id: Optional[str] = None, indent: int = 4, + update_baseline: bool = False, ): super().__init__() self.pm = plugin_manager @@ -106,6 +109,8 @@ class Context(DriverHolder): else: self.run_id = run_id + self.update_baseline = update_baseline + self.drivers_path = str(path.resolve()) self._context_data: Dict[str, dict] = {'Validation': {}, 'Outputs': {}, 'Definitions': {}} self._load_definition_files(definitions) @@ -231,10 +236,10 @@ class Context(DriverHolder): ) # type: ignore # Hook results will always be either size 1 or 0 - if len(hook_results) == 0: + if hook_results is None: repl_value = '' else: - repl_value = hook_results[0] + repl_value = hook_results if repl_value == '': self.logger.info( @@ -350,6 +355,7 @@ def get_default_context( logger: Optional[Logger] = None, run_id: Optional[str] = None, indent: int = 4, + update_baseline: bool = False, ) -> Context: ''' Gets the default context, creating a new one if necessary. @@ -371,6 +377,7 @@ def get_default_context( logger: An optional logger instance. If None, one will be created with the NullHandler. run_id: A string that uniquely identifies the run of Quilla. + update_baseline: Whether the VisualParity baselines should be updated or not Returns Application context shared for the entire application @@ -391,5 +398,6 @@ def get_default_context( logger, run_id, indent, + update_baseline ) return Context.default_context diff --git a/src/quilla/hookspecs/__init__.py b/src/quilla/hookspecs/__init__.py index 6ff2486..08d1738 100644 --- a/src/quilla/hookspecs/__init__.py +++ b/src/quilla/hookspecs/__init__.py @@ -1,3 +1,21 @@ -import pluggy +# flake8: noqa -hookspec = pluggy.HookspecMarker('quilla') +from quilla.hookspecs._hookspec import hookspec + +from .configuration import ( + quilla_addopts, + quilla_configure, + quilla_configure_logger, + quilla_prevalidate, + quilla_resolve_enum_from_name +) +from .reports import ( + quilla_postvalidate, +) +from .steps import ( + quilla_context_obj, + quilla_get_baseline_uri, + quilla_get_visualparity_baseline, + quilla_step_factory_selector, + quilla_store_image +) diff --git a/src/quilla/hookspecs/_hookspec.py b/src/quilla/hookspecs/_hookspec.py new file mode 100644 index 0000000..6ff2486 --- /dev/null +++ b/src/quilla/hookspecs/_hookspec.py @@ -0,0 +1,3 @@ +import pluggy + +hookspec = pluggy.HookspecMarker('quilla') diff --git a/src/quilla/hookspecs/configuration.py b/src/quilla/hookspecs/configuration.py index 7d967be..24315a4 100644 --- a/src/quilla/hookspecs/configuration.py +++ b/src/quilla/hookspecs/configuration.py @@ -14,7 +14,7 @@ from typing import ( Optional, ) -from quilla.hookspecs import hookspec +from quilla.hookspecs._hookspec import hookspec from quilla.ctx import Context from quilla.ui_validation import QuillaTest diff --git a/src/quilla/hookspecs/reports.py b/src/quilla/hookspecs/reports.py index a8c208b..8a3e1e4 100644 --- a/src/quilla/hookspecs/reports.py +++ b/src/quilla/hookspecs/reports.py @@ -3,7 +3,7 @@ Hooks related to outputs and reports ''' from quilla.ctx import Context -from quilla.hookspecs import hookspec +from quilla.hookspecs._hookspec import hookspec from quilla.reports.report_summary import ReportSummary diff --git a/src/quilla/hookspecs/steps.py b/src/quilla/hookspecs/steps.py index 4253259..cbf2d7c 100644 --- a/src/quilla/hookspecs/steps.py +++ b/src/quilla/hookspecs/steps.py @@ -9,7 +9,7 @@ from typing import ( Optional ) -from quilla.hookspecs import hookspec +from quilla.hookspecs._hookspec import hookspec from quilla.ctx import Context from quilla.common.enums import ( UITestActions, @@ -93,6 +93,23 @@ def quilla_store_image( ''' +@hookspec(firstresult=True) +def quilla_get_baseline_uri(ctx: Context, run_id: str, baseline_id: str) -> Optional[str]: + ''' + A hook to allow plugins to retrieve some URI for the image associated with + the baseline ID. + + Args: + ctx: The runtime context of the application + run_id: The run ID for the current run, in case the plugin tracks + baselines for each run + baseline_id: The unique ID for the baseline image + + Returns: + An identifier that can locate the baseline image + ''' + + @hookspec(firstresult=True) def quilla_get_visualparity_baseline(ctx: Context, baseline_id: str) -> Optional[bytes]: ''' diff --git a/src/quilla/plugins.py b/src/quilla/plugins/__init__.py similarity index 84% rename from src/quilla/plugins.py rename to src/quilla/plugins/__init__.py index 9e2c526..bee4c64 100644 --- a/src/quilla/plugins.py +++ b/src/quilla/plugins/__init__.py @@ -7,6 +7,8 @@ import pluggy from quilla import hookspecs +from .local_storage import LocalStorage + _hookimpl = pluggy.HookimplMarker('quilla') @@ -68,7 +70,11 @@ def _get_uiconf_plugins(pm: pluggy.PluginManager, root: Path, logger: Logger): _load_hooks_from_module(pm, uiconf_module, logger) -def _load_hooks_from_module(pm: pluggy.PluginManager, module, logger: Logger): +def _load_hooks_from_module( + pm: pluggy.PluginManager, + module, + logger: Logger, +): ''' Load a module into the given plugin manager object by finding all methods in the module that start with the `quilla_` prefix @@ -88,7 +94,12 @@ def _load_hooks_from_module(pm: pluggy.PluginManager, module, logger: Logger): setattr(module, hook, hook_function) logger.debug('Loading all discovered hooks into the plugin manager') - pm.register(module) + + # If it is callable, it is a class object not a module + if callable(module): + pm.register(module()) + else: + pm.register(module) def _load_entrypoint_plugins(pm: pluggy.PluginManager, logger: Logger): @@ -104,7 +115,16 @@ def _load_entrypoint_plugins(pm: pluggy.PluginManager, logger: Logger): entry_point.name ) logger.debug('Module encountered %s', e, exc_info=True) - pass + + +def _load_bundled_plugins(pm: pluggy.PluginManager, logger: Logger): + bundled_plugins = [ + LocalStorage, + ] + + for plugin in bundled_plugins: + logger.debug('Loading plugin module %s', plugin.__name__) + _load_hooks_from_module(pm, plugin, logger) def get_plugin_manager(path: str, logger: Logger) -> pluggy.PluginManager: @@ -119,9 +139,13 @@ def get_plugin_manager(path: str, logger: Logger) -> pluggy.PluginManager: a configured PluginManager instance with all plugins already loaded ''' pm = pluggy.PluginManager('quilla') + pm.add_hookspecs(hookspecs) logger.debug('Loading dummy hooks into plugin manager') pm.register(_DummyHooks) + logger.debug('Loading bundled plugins') + _load_bundled_plugins(pm, logger) + logger.debug('Loading entrypoint plugins') _load_entrypoint_plugins(pm, logger) diff --git a/src/quilla/plugins/base_storage.py b/src/quilla/plugins/base_storage.py new file mode 100644 index 0000000..108c5b0 --- /dev/null +++ b/src/quilla/plugins/base_storage.py @@ -0,0 +1,202 @@ +''' +This module contains a base class that can be useful when defining new +storage plugins, as there are some behaviours that will be shared among +the configuration objects +''' + +from abc import ( + ABC, + abstractproperty, + abstractmethod +) +from typing import ( + Optional +) + +from quilla.ctx import Context +from quilla.common.enums import VisualParityImageType + + +class BaseStorage(ABC): + @abstractproperty + def is_enabled(self) -> bool: + ''' + A property to be used to determine if the plugin is configured and + therefore able to run + + Returns: + True if the plugin is configured, False otherwise + ''' + + @abstractmethod + def find_image_by_baseline(self, baseline_id: str) -> bytes: + ''' + Searches the defined storage method for the image matching some + baseline ID + + Args: + baseline_id: The unique ID to search for. It is assumed every + baseline ID is unique regardless of what test is requesting it. + + Returns: + The bytes representation of the stored image if found, or an empty + bytes object if the image is not found. + ''' + + @abstractmethod + def cleanup_reports(self): + ''' + Searches for reports that match some cleanup criteria + ''' + + @abstractmethod + def store_treatment_image( + self, + run_id: str, + baseline_id: str, + treatment: bytes, + ) -> str: + ''' + Stores a treatment image within the storage mechanism enabled by the plugin + + Args: + run_id: The run ID of the current Quilla run, to version the treatment images + baseline_id: The ID of the baseline that this treatment image is associated + with + treatment: The image data in bytes + + Returns: + An identifier that can locate the newly stored treatment image + ''' + + @abstractmethod + def store_baseline_image( + self, + run_id: str, + baseline_id: str, + baseline: bytes, + ) -> str: + ''' + Stores a baseline image under the given baseline_id. + + This function should be used to update the current baseline + image, or create a new one if the baseline did not previously exist. + + The run ID is passed in case a plugin would like to use it to version and + store previous image baselines + + Args: + run_id: The ID of the current run of Quilla + baseline_id: A unique identifier for the image + baseline: The image data in bytes + + Returns: + A URI for the new baseline image + ''' + + @abstractmethod + def make_baseline_uri( + self, + run_id: str, + baseline_id: str + ) -> str: + ''' + Generates a baseline URI for the current run given the baseline_id of the image. + + It is recommended that plugins create a clone of the baseline image + when generating the URI so that the returned URI will uniquely identify + the baseline that was used for the associated run ID. This ensures that + even if the baseline image is updated, the report is still meaningful. + + Args: + run_id: The unique ID identifying the current run + baseline_id: The unique identifier for the image + + Returns: + A URI that can locate the baseline image used for the given run + ''' + + def get_image(self, baseline_id: str) -> Optional[bytes]: + ''' + Determines if the plugin should run, and if so searches for the image + with the specified baseline ID and returns the byte data for it + + Args: + baseline_id: The unique ID to search for + + Returns: + None if the plugin is not enabled, a ``bytes`` representation + of the image if it is. If no baseline image is found, returns an + empty byte string + ''' + if not self.is_enabled: + return None + + return self.find_image_by_baseline(baseline_id) + + def get_baseline_uri(self, run_id: str, baseline_id: str) -> Optional[str]: + ''' + Retrieves the URI for the baseline image + + Args: + run_id: The unique ID for the current run of Quilla + baseline_id: The unique ID for the baseline image + + Returns: + None if the plugin is not enabled, a string URI if it is + ''' + if not self.is_enabled: + return None + + return self.make_baseline_uri(run_id, baseline_id) + + def quilla_store_image( + self, + ctx: Context, + baseline_id: str, + image_bytes: bytes, + image_type: VisualParityImageType, + ) -> Optional[str]: + ''' + Stores a given image based on its type and possibly the run ID + + Args: + ctx: The runtime context for Quilla + baseline_id: The unique identifier for the image + image_bytes: The byte data for the image + image_type: The kind of image that is being stored + + Returns: + A URI for the image that was stored, or None if the plugin + is not enabled. The URI might be the empty string if the + image type is not supported + ''' + if not self.is_enabled: + return None + + run_id = ctx.run_id + + image_uri = '' + + function_selector = { + VisualParityImageType.TREATMENT: self.store_treatment_image, + VisualParityImageType.BASELINE: self.store_baseline_image, + } + + store_image_function = function_selector[image_type] + + image_uri = store_image_function( + run_id, + baseline_id, + image_bytes + ) + + self.cleanup_reports() + + return image_uri + + def quilla_get_baseline_uri(self, run_id: str, baseline_id: str) -> Optional[str]: + return self.get_baseline_uri(run_id, baseline_id) + + def quilla_get_visualparity_baseline(self, baseline_id: str) -> Optional[bytes]: + return self.get_image(baseline_id) diff --git a/src/quilla/plugins/local_storage.py b/src/quilla/plugins/local_storage.py new file mode 100644 index 0000000..ff537cf --- /dev/null +++ b/src/quilla/plugins/local_storage.py @@ -0,0 +1,129 @@ +''' +A plugin to add LocalStorage functionality for the VisualParity plugin. +''' + + +from argparse import ( + ArgumentParser, + Namespace, +) +from pathlib import Path +from typing import ( + Optional, + cast +) + +from .base_storage import BaseStorage + + +class LocalStorage(BaseStorage): + baseline_directory: Optional[Path] + runs_directory: Optional[Path] + + def __init__( + self, + storage_directory: Optional[str] = None + ): + if storage_directory is None: + self.baseline_directory = None + self.runs_directory = None + return + + self.configure(storage_directory) + + def configure(self, storage_directory: str): + ''' + Initialize all the required data + ''' + baseline_path = Path(storage_directory) + self.baseline_directory = baseline_path / 'baselines' + self.runs_directory = baseline_path / 'runs' + self.runs_directory.mkdir(exist_ok=True) + self.baseline_directory.mkdir(exist_ok=True) + + @property + def is_enabled(self) -> bool: + return self.baseline_directory is not None + + def run_path(self, run_id: str) -> Path: + path = cast(Path, self.runs_directory) / run_id + + path.mkdir(exist_ok=True) + + return path + + def store_baseline_image(self, run_id: str, baseline_id: str, baseline: bytes) -> str: + baseline_path = cast(Path, self.baseline_directory) / f'{baseline_id}.png' + + snapshot_path = baseline_path.parent / 'snapshots' / f'{baseline_id}_{run_id}.png' + + baseline_path.touch() + + baseline_path.write_bytes(baseline) + + snapshot_path.touch() + snapshot_path.write_bytes(baseline) + + return baseline_path.absolute().as_uri() + + def store_treatment_image( + self, + run_id: str, + baseline_id: str, + treatment: bytes + ) -> str: + + run_path = self.run_path(run_id) + + image_path = run_path / f'{baseline_id}_treatment.png' + + image_path.write_bytes(treatment) + + return image_path.absolute().as_uri() + + def find_image_by_baseline(self, baseline_id: str) -> bytes: + image_path = cast(Path, self.baseline_directory) / f'{baseline_id}.png' + + if not image_path.exists(): + return b'' + + return image_path.read_bytes() + + def make_baseline_uri(self, run_id: str, baseline_id: str) -> str: + image_data = self.find_image_by_baseline(baseline_id) + + run_path = self.run_path(run_id) + + image_path = run_path / f'{baseline_id}.png' + + image_path.touch() + + image_path.write_bytes(image_data) + + return image_path.absolute().as_uri() + + def cleanup_reports(self): + ''' + Method left blank, as no cleanup is provided for LocalStorage since + users are expected to have granular control over their own filesystems + ''' + + def quilla_addopts(self, parser: ArgumentParser): + ''' + Using the Quilla hook to add a new group of CLI args to the parser + ''' + + ls_group = parser.add_argument_group(title='Local Storage Options') + + ls_group.add_argument( + '--image-directory', + dest='image_dir', + action='store', + default=None, + help='The directory that should be used for the LocalStorage ' + 'plugin to store VisualParity images' + ) + + def quilla_configure(self, args: Namespace): + if args.image_dir is not None: + self.configure(args.image_dir) diff --git a/src/quilla/reports/visual_parity_report.py b/src/quilla/reports/visual_parity_report.py index 39dc604..32b743e 100644 --- a/src/quilla/reports/visual_parity_report.py +++ b/src/quilla/reports/visual_parity_report.py @@ -54,9 +54,9 @@ class VisualParityReport(ValidationReport): success=success, msg=msg ) - self.baseline_id = baseline_id, - self.baseline_image_uri = baseline_image_uri, - self.treatment_image_uri = treatment_image_uri, + self.baseline_id = baseline_id + self.baseline_image_uri = baseline_image_uri + self.treatment_image_uri = treatment_image_uri self.delta_image_uri = delta_image_uri def to_dict(self): diff --git a/src/quilla/steps/base_steps.py b/src/quilla/steps/base_steps.py index b1df529..021e650 100644 --- a/src/quilla/steps/base_steps.py +++ b/src/quilla/steps/base_steps.py @@ -121,7 +121,7 @@ class BaseStep(DriverHolder, EnumResolver): ) def _verify_target(self): - if self.target is None: + if self._target is None: raise FailedStepException(f'No specified target for "{self.action.value}" action') @property diff --git a/src/quilla/steps/validations/xpath.py b/src/quilla/steps/validations/xpath.py index 6148abd..61fd1c2 100644 --- a/src/quilla/steps/validations/xpath.py +++ b/src/quilla/steps/validations/xpath.py @@ -212,24 +212,70 @@ class XPathValidation(BaseValidation): baseline_id = self.parameters['baselineID'] + if self.ctx.update_baseline: + result = self.ctx.pm.hook.quilla_store_image( + ctx=self.ctx, + baseline_id=baseline_id, + image_bytes=self.element.screenshot_as_png, + image_type=VisualParityImageType.BASELINE + ) + + if result is None: + return VisualParityReport( + success=False, + target=self._target, + browser_name=self.driver.name, + baseline_id=baseline_id, + msg='No baseline storage mechanism configured' + ) + + baseline_uri = result + + if baseline_uri == '': + return VisualParityReport( + success=False, + target=self._target, + browser_name=self.driver.name, + baseline_id=baseline_id, + msg='Unable to update the baseline image' + ) + + return VisualParityReport( + success=True, + target=self._target, + browser_name=self.driver.name, + baseline_id=baseline_id, + baseline_image_uri=baseline_uri, + msg='Successfully updated baseline URI' + ) + treatment_image_bytes = self.element.screenshot_as_png treatment_image = Image.open(BytesIO(treatment_image_bytes)) - plugin_result: List = self.ctx.pm.hook.quilla_get_visualparity_baseline( + plugin_result = self.ctx.pm.hook.quilla_get_visualparity_baseline( ctx=self.ctx, baseline_id=baseline_id ) + if plugin_result is None: + return VisualParityReport( + success=False, + target=self._target, + browser_name=self.driver.name, + baseline_id=baseline_id, + msg='No baseline storage mechanism configured' + ) + if len(plugin_result) == 0: return VisualParityReport( success=False, target=self._target, browser_name=self.driver.name, baseline_id=baseline_id, - msg='No baseline storage mechanism configured, or no baseline found' + msg='No baseline image found' ) - baseline_image_bytes, baseline_uri = plugin_result[0] + baseline_image_bytes = plugin_result baseline_image = Image.open(BytesIO(baseline_image_bytes)) success = baseline_image == treatment_image # Run the comparison with Pillow @@ -249,6 +295,12 @@ class XPathValidation(BaseValidation): image_type=VisualParityImageType.TREATMENT, ) + baseline_uri = self.ctx.pm.hook.quilla_get_baseline_uri( + ctx=self.ctx, + run_id=self.ctx.run_id, + baseline_id=baseline_id + ) + return VisualParityReport( success=False, target=self._target, diff --git a/tests/integration/test_quilla_plugins_work.json b/tests/integration/test_quilla_plugins_work.json deleted file mode 100644 index f68bbe7..0000000 --- a/tests/integration/test_quilla_plugins_work.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "markers": ["integration", "quilla", "slow"], - "targetBrowsers": ["Firefox"], - "path": "https://bing.com", - "steps": [ - { - "action": "OutputValue", - "target": "${{ Driver.title }}", - "parameters": { - "source": "Literal", - "outputName": "site_title" - } - }, - { - "action": "Validate", - "target": "${{ Validation.site_title }}", - "type": "URL", - "state": "Contains" - } - ] -} diff --git a/tests/integration/test_visual_parity.json b/tests/integration/test_visual_parity.json new file mode 100644 index 0000000..a8dc959 --- /dev/null +++ b/tests/integration/test_visual_parity.json @@ -0,0 +1,79 @@ +{ + "definitions": { + "RiddleInput": "//input[@id='r1Input']", + "PasswordInput": "//input[@id='r2Input']", + "MerchantInput": "//input[@id='r3Input']", + "RiddleSubmitButton": "//button[@id='r1Btn']", + "PasswordSubmitButton": "//button[@id='r2Butn']", + "MerchantSubmitButton": "//button[@id='r3Butn']", + "TrialSubmitButton": "//button[@id='checkButn']", + "PasswordBanner": "//div[@id='passwordBanner']", + "TrialCompleteBanner": "//div[@id='trialCompleteBanner']" + }, + "targetBrowsers": [ + "Firefox" + ], + "path": "https://techstepacademy.com/trial-of-the-stones", + "steps": [ + { + "action": "SendKeys", + "target": "${{ Definitions.RiddleInput }}", + "parameters": { + "data": "rock" + } + }, + { + "action": "Click", + "target": "${{ Definitions.RiddleSubmitButton }}" + }, + { + "action": "OutputValue", + "target": "${{ Definitions.PasswordBanner }}", + "parameters": { + "source": "XPathText", + "outputName": "trialPassword" + } + }, + { + "action": "SendKeys", + "target": "${{ Definitions.PasswordInput }}", + "parameters": { + "data": "${{ Validation.trialPassword }}" + } + }, + { + "action": "Click", + "target": "${{ Definitions.PasswordSubmitButton }}" + }, + { + "action": "SendKeys", + "target": "${{ Definitions.MerchantInput }}", + "parameters": { + "data": "Jessica" + } + }, + { + "action": "Click", + "target": "${{ Definitions.MerchantSubmitButton }}" + }, + { + "action": "Click", + "target": "${{ Definitions.TrialSubmitButton }}" + }, + { + "action": "Validate", + "type": "XPath", + "state": "Visible", + "target": "${{ Definitions.TrialCompleteBanner }}" + }, + { + "action": "Validate", + "type": "XPath", + "state": "VisualParity", + "target": "${{ Definitions.TrialCompleteBanner }}", + "parameters": { + "baselineID": "TrialCompleteBanner" + } + } + ] +}