new: added LocalStorage plugin for VisualParity (#42)

This commit is contained in:
Natalia Maximo 2021-07-15 14:54:05 -04:00 коммит произвёл GitHub
Родитель 13292d241e
Коммит 9cd8a44b46
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
18 изменённых файлов: 566 добавлений и 44 удалений

Двоичные данные
images/baselines/TrialCompleteBanner.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 1.9 KiB

Просмотреть файл

@ -21,5 +21,5 @@ markers =
quilla: Marks tests written to be executed with Quilla
integration: Marks an integration test.
testpaths = tests
addopts = --cov=src --cov-report term-missing -p no:quilla -n auto
addopts = --cov=src --cov-report term-missing -p no:quilla -n auto --quilla-opts="--image-directory ./images"
python_classes = *Tests

Просмотреть файл

@ -65,6 +65,16 @@ def make_parser() -> argparse.ArgumentParser: # pragma: no cover
default=None,
help='A run ID for quilla, if manually passed in. '
'Used to set many quilla tests to have the same run ID '
'If no run ID is manually passed in, it will be auto-generated'
)
config_group.add_argument(
'-u',
'--update-baseline',
dest='update_baseline',
action='store_true',
help='Used to update the baseline images for VisualParity. '
'Different plugins define different behaviour, so this is not '
'necessarily a lossless operation.'
)
config_group.add_argument(
'-d',
@ -282,10 +292,11 @@ def setup_context(args: List[str], plugin_root: str = '.') -> Context:
logger=logger,
run_id=parsed_args.run_id,
indent=parsed_args.indent,
update_baseline=parsed_args.update_baseline
)
logger.info('Running "quilla_configure" hook')
pm.hook.quilla_configure(ctx=ctx, args=args)
pm.hook.quilla_configure(ctx=ctx, args=parsed_args)
return ctx

Просмотреть файл

@ -81,7 +81,7 @@ class EnumResolver:
if ctx is not None:
resolved_plugin_value = ctx.pm.hook.quilla_resolve_enum_from_name(name=name, enum=enum)
if len(resolved_plugin_value) > 0:
return resolved_plugin_value[0]
if resolved_plugin_value is not None:
return resolved_plugin_value
raise EnumValueNotFoundException(name, enum)

Просмотреть файл

@ -41,6 +41,7 @@ class Context(DriverHolder):
no_sandbox: Whether to pass the '--no-sandbox' arg to Chrome and Edge
logger: An optional configured logger instance.
run_id: A string that uniquely identifies the run of Quilla.
update_baseline: Whether the VisualParity baselines should be updated or not
Attributes:
@ -58,6 +59,7 @@ class Context(DriverHolder):
one with the default logger.
run_id: A string that uniquely identifies the run of Quilla.
pretty_print_indent: How many spaces to use for indentation when pretty-printing the output
update_baseline: Whether the VisualParity baselines should be updated or not
'''
default_context: Optional['Context'] = None
_drivers_path: str
@ -83,6 +85,7 @@ class Context(DriverHolder):
logger: Optional[Logger] = None,
run_id: Optional[str] = None,
indent: int = 4,
update_baseline: bool = False,
):
super().__init__()
self.pm = plugin_manager
@ -106,6 +109,8 @@ class Context(DriverHolder):
else:
self.run_id = run_id
self.update_baseline = update_baseline
self.drivers_path = str(path.resolve())
self._context_data: Dict[str, dict] = {'Validation': {}, 'Outputs': {}, 'Definitions': {}}
self._load_definition_files(definitions)
@ -231,10 +236,10 @@ class Context(DriverHolder):
) # type: ignore
# Hook results will always be either size 1 or 0
if len(hook_results) == 0:
if hook_results is None:
repl_value = ''
else:
repl_value = hook_results[0]
repl_value = hook_results
if repl_value == '':
self.logger.info(
@ -350,6 +355,7 @@ def get_default_context(
logger: Optional[Logger] = None,
run_id: Optional[str] = None,
indent: int = 4,
update_baseline: bool = False,
) -> Context:
'''
Gets the default context, creating a new one if necessary.
@ -371,6 +377,7 @@ def get_default_context(
logger: An optional logger instance. If None, one will be created
with the NullHandler.
run_id: A string that uniquely identifies the run of Quilla.
update_baseline: Whether the VisualParity baselines should be updated or not
Returns
Application context shared for the entire application
@ -391,5 +398,6 @@ def get_default_context(
logger,
run_id,
indent,
update_baseline
)
return Context.default_context

Просмотреть файл

@ -1,3 +1,21 @@
import pluggy
# flake8: noqa
hookspec = pluggy.HookspecMarker('quilla')
from quilla.hookspecs._hookspec import hookspec
from .configuration import (
quilla_addopts,
quilla_configure,
quilla_configure_logger,
quilla_prevalidate,
quilla_resolve_enum_from_name
)
from .reports import (
quilla_postvalidate,
)
from .steps import (
quilla_context_obj,
quilla_get_baseline_uri,
quilla_get_visualparity_baseline,
quilla_step_factory_selector,
quilla_store_image
)

Просмотреть файл

@ -0,0 +1,3 @@
import pluggy
hookspec = pluggy.HookspecMarker('quilla')

Просмотреть файл

@ -14,7 +14,7 @@ from typing import (
Optional,
)
from quilla.hookspecs import hookspec
from quilla.hookspecs._hookspec import hookspec
from quilla.ctx import Context
from quilla.ui_validation import QuillaTest

Просмотреть файл

@ -3,7 +3,7 @@ Hooks related to outputs and reports
'''
from quilla.ctx import Context
from quilla.hookspecs import hookspec
from quilla.hookspecs._hookspec import hookspec
from quilla.reports.report_summary import ReportSummary

Просмотреть файл

@ -9,7 +9,7 @@ from typing import (
Optional
)
from quilla.hookspecs import hookspec
from quilla.hookspecs._hookspec import hookspec
from quilla.ctx import Context
from quilla.common.enums import (
UITestActions,
@ -93,6 +93,23 @@ def quilla_store_image(
'''
@hookspec(firstresult=True)
def quilla_get_baseline_uri(ctx: Context, run_id: str, baseline_id: str) -> Optional[str]:
'''
A hook to allow plugins to retrieve some URI for the image associated with
the baseline ID.
Args:
ctx: The runtime context of the application
run_id: The run ID for the current run, in case the plugin tracks
baselines for each run
baseline_id: The unique ID for the baseline image
Returns:
An identifier that can locate the baseline image
'''
@hookspec(firstresult=True)
def quilla_get_visualparity_baseline(ctx: Context, baseline_id: str) -> Optional[bytes]:
'''

Просмотреть файл

@ -7,6 +7,8 @@ import pluggy
from quilla import hookspecs
from .local_storage import LocalStorage
_hookimpl = pluggy.HookimplMarker('quilla')
@ -68,7 +70,11 @@ def _get_uiconf_plugins(pm: pluggy.PluginManager, root: Path, logger: Logger):
_load_hooks_from_module(pm, uiconf_module, logger)
def _load_hooks_from_module(pm: pluggy.PluginManager, module, logger: Logger):
def _load_hooks_from_module(
pm: pluggy.PluginManager,
module,
logger: Logger,
):
'''
Load a module into the given plugin manager object by finding all
methods in the module that start with the `quilla_` prefix
@ -88,6 +94,11 @@ def _load_hooks_from_module(pm: pluggy.PluginManager, module, logger: Logger):
setattr(module, hook, hook_function)
logger.debug('Loading all discovered hooks into the plugin manager')
# If it is callable, it is a class object not a module
if callable(module):
pm.register(module())
else:
pm.register(module)
@ -104,7 +115,16 @@ def _load_entrypoint_plugins(pm: pluggy.PluginManager, logger: Logger):
entry_point.name
)
logger.debug('Module encountered %s', e, exc_info=True)
pass
def _load_bundled_plugins(pm: pluggy.PluginManager, logger: Logger):
bundled_plugins = [
LocalStorage,
]
for plugin in bundled_plugins:
logger.debug('Loading plugin module %s', plugin.__name__)
_load_hooks_from_module(pm, plugin, logger)
def get_plugin_manager(path: str, logger: Logger) -> pluggy.PluginManager:
@ -119,9 +139,13 @@ def get_plugin_manager(path: str, logger: Logger) -> pluggy.PluginManager:
a configured PluginManager instance with all plugins already loaded
'''
pm = pluggy.PluginManager('quilla')
pm.add_hookspecs(hookspecs)
logger.debug('Loading dummy hooks into plugin manager')
pm.register(_DummyHooks)
logger.debug('Loading bundled plugins')
_load_bundled_plugins(pm, logger)
logger.debug('Loading entrypoint plugins')
_load_entrypoint_plugins(pm, logger)

Просмотреть файл

@ -0,0 +1,202 @@
'''
This module contains a base class that can be useful when defining new
storage plugins, as there are some behaviours that will be shared among
the configuration objects
'''
from abc import (
ABC,
abstractproperty,
abstractmethod
)
from typing import (
Optional
)
from quilla.ctx import Context
from quilla.common.enums import VisualParityImageType
class BaseStorage(ABC):
@abstractproperty
def is_enabled(self) -> bool:
'''
A property to be used to determine if the plugin is configured and
therefore able to run
Returns:
True if the plugin is configured, False otherwise
'''
@abstractmethod
def find_image_by_baseline(self, baseline_id: str) -> bytes:
'''
Searches the defined storage method for the image matching some
baseline ID
Args:
baseline_id: The unique ID to search for. It is assumed every
baseline ID is unique regardless of what test is requesting it.
Returns:
The bytes representation of the stored image if found, or an empty
bytes object if the image is not found.
'''
@abstractmethod
def cleanup_reports(self):
'''
Searches for reports that match some cleanup criteria
'''
@abstractmethod
def store_treatment_image(
self,
run_id: str,
baseline_id: str,
treatment: bytes,
) -> str:
'''
Stores a treatment image within the storage mechanism enabled by the plugin
Args:
run_id: The run ID of the current Quilla run, to version the treatment images
baseline_id: The ID of the baseline that this treatment image is associated
with
treatment: The image data in bytes
Returns:
An identifier that can locate the newly stored treatment image
'''
@abstractmethod
def store_baseline_image(
self,
run_id: str,
baseline_id: str,
baseline: bytes,
) -> str:
'''
Stores a baseline image under the given baseline_id.
This function should be used to update the current baseline
image, or create a new one if the baseline did not previously exist.
The run ID is passed in case a plugin would like to use it to version and
store previous image baselines
Args:
run_id: The ID of the current run of Quilla
baseline_id: A unique identifier for the image
baseline: The image data in bytes
Returns:
A URI for the new baseline image
'''
@abstractmethod
def make_baseline_uri(
self,
run_id: str,
baseline_id: str
) -> str:
'''
Generates a baseline URI for the current run given the baseline_id of the image.
It is recommended that plugins create a clone of the baseline image
when generating the URI so that the returned URI will uniquely identify
the baseline that was used for the associated run ID. This ensures that
even if the baseline image is updated, the report is still meaningful.
Args:
run_id: The unique ID identifying the current run
baseline_id: The unique identifier for the image
Returns:
A URI that can locate the baseline image used for the given run
'''
def get_image(self, baseline_id: str) -> Optional[bytes]:
'''
Determines if the plugin should run, and if so searches for the image
with the specified baseline ID and returns the byte data for it
Args:
baseline_id: The unique ID to search for
Returns:
None if the plugin is not enabled, a ``bytes`` representation
of the image if it is. If no baseline image is found, returns an
empty byte string
'''
if not self.is_enabled:
return None
return self.find_image_by_baseline(baseline_id)
def get_baseline_uri(self, run_id: str, baseline_id: str) -> Optional[str]:
'''
Retrieves the URI for the baseline image
Args:
run_id: The unique ID for the current run of Quilla
baseline_id: The unique ID for the baseline image
Returns:
None if the plugin is not enabled, a string URI if it is
'''
if not self.is_enabled:
return None
return self.make_baseline_uri(run_id, baseline_id)
def quilla_store_image(
self,
ctx: Context,
baseline_id: str,
image_bytes: bytes,
image_type: VisualParityImageType,
) -> Optional[str]:
'''
Stores a given image based on its type and possibly the run ID
Args:
ctx: The runtime context for Quilla
baseline_id: The unique identifier for the image
image_bytes: The byte data for the image
image_type: The kind of image that is being stored
Returns:
A URI for the image that was stored, or None if the plugin
is not enabled. The URI might be the empty string if the
image type is not supported
'''
if not self.is_enabled:
return None
run_id = ctx.run_id
image_uri = ''
function_selector = {
VisualParityImageType.TREATMENT: self.store_treatment_image,
VisualParityImageType.BASELINE: self.store_baseline_image,
}
store_image_function = function_selector[image_type]
image_uri = store_image_function(
run_id,
baseline_id,
image_bytes
)
self.cleanup_reports()
return image_uri
def quilla_get_baseline_uri(self, run_id: str, baseline_id: str) -> Optional[str]:
return self.get_baseline_uri(run_id, baseline_id)
def quilla_get_visualparity_baseline(self, baseline_id: str) -> Optional[bytes]:
return self.get_image(baseline_id)

Просмотреть файл

@ -0,0 +1,129 @@
'''
A plugin to add LocalStorage functionality for the VisualParity plugin.
'''
from argparse import (
ArgumentParser,
Namespace,
)
from pathlib import Path
from typing import (
Optional,
cast
)
from .base_storage import BaseStorage
class LocalStorage(BaseStorage):
baseline_directory: Optional[Path]
runs_directory: Optional[Path]
def __init__(
self,
storage_directory: Optional[str] = None
):
if storage_directory is None:
self.baseline_directory = None
self.runs_directory = None
return
self.configure(storage_directory)
def configure(self, storage_directory: str):
'''
Initialize all the required data
'''
baseline_path = Path(storage_directory)
self.baseline_directory = baseline_path / 'baselines'
self.runs_directory = baseline_path / 'runs'
self.runs_directory.mkdir(exist_ok=True)
self.baseline_directory.mkdir(exist_ok=True)
@property
def is_enabled(self) -> bool:
return self.baseline_directory is not None
def run_path(self, run_id: str) -> Path:
path = cast(Path, self.runs_directory) / run_id
path.mkdir(exist_ok=True)
return path
def store_baseline_image(self, run_id: str, baseline_id: str, baseline: bytes) -> str:
baseline_path = cast(Path, self.baseline_directory) / f'{baseline_id}.png'
snapshot_path = baseline_path.parent / 'snapshots' / f'{baseline_id}_{run_id}.png'
baseline_path.touch()
baseline_path.write_bytes(baseline)
snapshot_path.touch()
snapshot_path.write_bytes(baseline)
return baseline_path.absolute().as_uri()
def store_treatment_image(
self,
run_id: str,
baseline_id: str,
treatment: bytes
) -> str:
run_path = self.run_path(run_id)
image_path = run_path / f'{baseline_id}_treatment.png'
image_path.write_bytes(treatment)
return image_path.absolute().as_uri()
def find_image_by_baseline(self, baseline_id: str) -> bytes:
image_path = cast(Path, self.baseline_directory) / f'{baseline_id}.png'
if not image_path.exists():
return b''
return image_path.read_bytes()
def make_baseline_uri(self, run_id: str, baseline_id: str) -> str:
image_data = self.find_image_by_baseline(baseline_id)
run_path = self.run_path(run_id)
image_path = run_path / f'{baseline_id}.png'
image_path.touch()
image_path.write_bytes(image_data)
return image_path.absolute().as_uri()
def cleanup_reports(self):
'''
Method left blank, as no cleanup is provided for LocalStorage since
users are expected to have granular control over their own filesystems
'''
def quilla_addopts(self, parser: ArgumentParser):
'''
Using the Quilla hook to add a new group of CLI args to the parser
'''
ls_group = parser.add_argument_group(title='Local Storage Options')
ls_group.add_argument(
'--image-directory',
dest='image_dir',
action='store',
default=None,
help='The directory that should be used for the LocalStorage '
'plugin to store VisualParity images'
)
def quilla_configure(self, args: Namespace):
if args.image_dir is not None:
self.configure(args.image_dir)

Просмотреть файл

@ -54,9 +54,9 @@ class VisualParityReport(ValidationReport):
success=success,
msg=msg
)
self.baseline_id = baseline_id,
self.baseline_image_uri = baseline_image_uri,
self.treatment_image_uri = treatment_image_uri,
self.baseline_id = baseline_id
self.baseline_image_uri = baseline_image_uri
self.treatment_image_uri = treatment_image_uri
self.delta_image_uri = delta_image_uri
def to_dict(self):

Просмотреть файл

@ -121,7 +121,7 @@ class BaseStep(DriverHolder, EnumResolver):
)
def _verify_target(self):
if self.target is None:
if self._target is None:
raise FailedStepException(f'No specified target for "{self.action.value}" action')
@property

Просмотреть файл

@ -212,24 +212,70 @@ class XPathValidation(BaseValidation):
baseline_id = self.parameters['baselineID']
if self.ctx.update_baseline:
result = self.ctx.pm.hook.quilla_store_image(
ctx=self.ctx,
baseline_id=baseline_id,
image_bytes=self.element.screenshot_as_png,
image_type=VisualParityImageType.BASELINE
)
if result is None:
return VisualParityReport(
success=False,
target=self._target,
browser_name=self.driver.name,
baseline_id=baseline_id,
msg='No baseline storage mechanism configured'
)
baseline_uri = result
if baseline_uri == '':
return VisualParityReport(
success=False,
target=self._target,
browser_name=self.driver.name,
baseline_id=baseline_id,
msg='Unable to update the baseline image'
)
return VisualParityReport(
success=True,
target=self._target,
browser_name=self.driver.name,
baseline_id=baseline_id,
baseline_image_uri=baseline_uri,
msg='Successfully updated baseline URI'
)
treatment_image_bytes = self.element.screenshot_as_png
treatment_image = Image.open(BytesIO(treatment_image_bytes))
plugin_result: List = self.ctx.pm.hook.quilla_get_visualparity_baseline(
plugin_result = self.ctx.pm.hook.quilla_get_visualparity_baseline(
ctx=self.ctx,
baseline_id=baseline_id
)
if plugin_result is None:
return VisualParityReport(
success=False,
target=self._target,
browser_name=self.driver.name,
baseline_id=baseline_id,
msg='No baseline storage mechanism configured'
)
if len(plugin_result) == 0:
return VisualParityReport(
success=False,
target=self._target,
browser_name=self.driver.name,
baseline_id=baseline_id,
msg='No baseline storage mechanism configured, or no baseline found'
msg='No baseline image found'
)
baseline_image_bytes, baseline_uri = plugin_result[0]
baseline_image_bytes = plugin_result
baseline_image = Image.open(BytesIO(baseline_image_bytes))
success = baseline_image == treatment_image # Run the comparison with Pillow
@ -249,6 +295,12 @@ class XPathValidation(BaseValidation):
image_type=VisualParityImageType.TREATMENT,
)
baseline_uri = self.ctx.pm.hook.quilla_get_baseline_uri(
ctx=self.ctx,
run_id=self.ctx.run_id,
baseline_id=baseline_id
)
return VisualParityReport(
success=False,
target=self._target,

Просмотреть файл

@ -1,21 +0,0 @@
{
"markers": ["integration", "quilla", "slow"],
"targetBrowsers": ["Firefox"],
"path": "https://bing.com",
"steps": [
{
"action": "OutputValue",
"target": "${{ Driver.title }}",
"parameters": {
"source": "Literal",
"outputName": "site_title"
}
},
{
"action": "Validate",
"target": "${{ Validation.site_title }}",
"type": "URL",
"state": "Contains"
}
]
}

Просмотреть файл

@ -0,0 +1,79 @@
{
"definitions": {
"RiddleInput": "//input[@id='r1Input']",
"PasswordInput": "//input[@id='r2Input']",
"MerchantInput": "//input[@id='r3Input']",
"RiddleSubmitButton": "//button[@id='r1Btn']",
"PasswordSubmitButton": "//button[@id='r2Butn']",
"MerchantSubmitButton": "//button[@id='r3Butn']",
"TrialSubmitButton": "//button[@id='checkButn']",
"PasswordBanner": "//div[@id='passwordBanner']",
"TrialCompleteBanner": "//div[@id='trialCompleteBanner']"
},
"targetBrowsers": [
"Firefox"
],
"path": "https://techstepacademy.com/trial-of-the-stones",
"steps": [
{
"action": "SendKeys",
"target": "${{ Definitions.RiddleInput }}",
"parameters": {
"data": "rock"
}
},
{
"action": "Click",
"target": "${{ Definitions.RiddleSubmitButton }}"
},
{
"action": "OutputValue",
"target": "${{ Definitions.PasswordBanner }}",
"parameters": {
"source": "XPathText",
"outputName": "trialPassword"
}
},
{
"action": "SendKeys",
"target": "${{ Definitions.PasswordInput }}",
"parameters": {
"data": "${{ Validation.trialPassword }}"
}
},
{
"action": "Click",
"target": "${{ Definitions.PasswordSubmitButton }}"
},
{
"action": "SendKeys",
"target": "${{ Definitions.MerchantInput }}",
"parameters": {
"data": "Jessica"
}
},
{
"action": "Click",
"target": "${{ Definitions.MerchantSubmitButton }}"
},
{
"action": "Click",
"target": "${{ Definitions.TrialSubmitButton }}"
},
{
"action": "Validate",
"type": "XPath",
"state": "Visible",
"target": "${{ Definitions.TrialCompleteBanner }}"
},
{
"action": "Validate",
"type": "XPath",
"state": "VisualParity",
"target": "${{ Definitions.TrialCompleteBanner }}",
"parameters": {
"baselineID": "TrialCompleteBanner"
}
}
]
}