new: added blob storage plugin (#44)

.github/workflows/test-pipeline.yml

@@ -42,7 +42,7 @@ jobs:
         run: echo "JUNIT_XML_OUT=quilla-pytest-junit-RUN${{ env.GITHUB_RUN_NUMBER }}-$(date +'%Y-%m-%d').xml" >> $GITHUB_ENV
 
       - name: Run tests
-        run: pytest --junit-xml="${{ env.JUNIT_XML_OUT }}"
+        run: pytest --junit-xml="${{ env.JUNIT_XML_OUT }}" --quilla-opts="--connection-string ${{ secrets.CONNECTION_STRING }}"
 
       - name: Upload JUnit XML artifact
         if: ${{ always() }}

.pre-commit-config.yaml

@@ -12,7 +12,6 @@ repos:
       - id: check-merge-conflict
       - id: check-executables-have-shebangs
       - id: check-json
-      - id: pretty-format-json
       - id: debug-statements
       - id: detect-private-key
       - id: double-quote-string-fixer

docs/source/quilla.plugins.rst

@@ -0,0 +1,37 @@
+quilla.plugins package
+======================
+
+Submodules
+----------
+
+quilla.plugins.base\_storage module
+-----------------------------------
+
+.. automodule:: quilla.plugins.base_storage
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+quilla.plugins.blob\_storage module
+-----------------------------------
+
+.. automodule:: quilla.plugins.blob_storage
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+quilla.plugins.local\_storage module
+------------------------------------
+
+.. automodule:: quilla.plugins.local_storage
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: quilla.plugins
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/quilla.steps.rst

@@ -25,10 +25,11 @@ quilla.steps.steps module
    :undoc-members:
    :show-inheritance:
 
-quilla.steps.validations module
--------------------------------
 
-.. automodule:: quilla.steps.validations
-   :members:
-   :undoc-members:
-   :show-inheritance:
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   quilla.steps.validations

docs/source/quilla.steps.validations.rst

@@ -0,0 +1,37 @@
+quilla.steps.validations package
+================================
+
+Submodules
+----------
+
+quilla.steps.validations.url module
+-----------------------------------
+
+.. automodule:: quilla.steps.validations.url
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+quilla.steps.validations.validation\_factory module
+---------------------------------------------------
+
+.. automodule:: quilla.steps.validations.validation_factory
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+quilla.steps.validations.xpath module
+-------------------------------------
+
+.. automodule:: quilla.steps.validations.xpath
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: quilla.steps.validations
+   :members:
+   :undoc-members:
+   :show-inheritance:

setup.py

@@ -69,7 +69,8 @@ setup(
         'pluggy',
         'msedge-selenium-tools',
         'pydeepmerge',
-        'pillow'
+        'pillow',
+        'azure-storage-blob',
     ],
     tests_require=extra_dependencies['tests'],
     extras_require=extra_dependencies,

src/pytest_quilla/pytest_classes.py

@@ -68,8 +68,10 @@ class QuillaItem(pytest.Item):
         '''
         ctx = setup_context(
             [*self.config.getoption('--quilla-opts').split(), ''],
-            str(self.config.rootpath)
+            str(self.config.rootpath),
+            recreate_context=True
         )
+        ctx.logger.debug('Quilla options discovered: %s', self.config.getoption('--quilla-opts'))
         if not (
             '-i' in self.config.getoption('--quilla-opts') or
             '--run-id' in self.config.getoption('--quilla-opts')

src/quilla/__init__.py

@@ -234,13 +234,18 @@ def execute(ctx: Context) -> ReportSummary:
     return reports
 
 
-def setup_context(args: List[str], plugin_root: str = '.') -> Context:
+def setup_context(
+    args: List[str],
+    plugin_root: str = '.',
+    recreate_context: bool = False,
+) -> Context:
     '''
     Starts up the plugin manager, creates parser, parses args and sets up the application context
 
     Args:
         args: A list of cli options, such as sys.argv[1:]
         plugin_root: The directory used by the plugin manager to search for `uiconf.py` files
+        recreate_context: Whether the context should be recreated
     Returns:
         A runtime context configured by the hooks and the args
     '''
@@ -292,7 +297,8 @@ def setup_context(args: List[str], plugin_root: str = '.') -> Context:
         logger=logger,
         run_id=parsed_args.run_id,
         indent=parsed_args.indent,
-        update_baseline=parsed_args.update_baseline
+        update_baseline=parsed_args.update_baseline,
+        recreate_context=recreate_context,
     )
 
     logger.info('Running "quilla_configure" hook')

src/quilla/plugins/__init__.py

@@ -8,6 +8,7 @@ import pluggy
 from quilla import hookspecs
 
 from .local_storage import LocalStorage
+from .blob_storage import BlobStorage
 
 
 _hookimpl = pluggy.HookimplMarker('quilla')
@@ -120,6 +121,7 @@ def _load_entrypoint_plugins(pm: pluggy.PluginManager, logger: Logger):
 def _load_bundled_plugins(pm: pluggy.PluginManager, logger: Logger):
     bundled_plugins = [
         LocalStorage,
+        BlobStorage,
     ]
 
     for plugin in bundled_plugins:

src/quilla/plugins/base_storage.py

@@ -46,7 +46,9 @@ class BaseStorage(ABC):
     @abstractmethod
     def cleanup_reports(self):
         '''
-        Searches for reports that match some cleanup criteria
+        Searches for reports that match some cleanup criteria, and deletes them
+        if necessary. Not every storage plugin will implement logic for this function,
+        choosing instead to have all images exist indefinitely.
         '''
 
     @abstractmethod

src/quilla/plugins/blob_storage.py

@@ -0,0 +1,167 @@
+from typing import (
+    Optional,
+    cast,
+)
+from argparse import (
+    ArgumentParser,
+    Namespace
+)
+from datetime import datetime
+
+from azure.storage.blob import ContainerClient
+
+from .base_storage import BaseStorage
+
+
+class BlobStorage(BaseStorage):
+    _container_client: Optional[ContainerClient]
+    max_retention_days: int
+
+    def __init__(self):
+        self._container_client = None
+
+    def quilla_addopts(self, parser: ArgumentParser):
+        '''
+        Adds the appropriate CLI arguments
+
+        Args:
+            parser: The Quilla argument parser
+        '''
+        az_group = parser.add_argument_group(
+            title='Azure Blob Storage Options'
+        )
+
+        az_group.add_argument(
+            '--connection-string',
+            dest='connection_string',
+            action='store',
+            default=None,
+            help='A connection string for the azure storage account to be used'
+        )
+
+        az_group.add_argument(
+            '--container-name',
+            dest='container_name',
+            action='store',
+            default='quilla',
+            help='The name of the container that will be used for storing Quilla images'
+        )
+
+        az_group.add_argument(
+            '--retention-days',
+            dest='retention_days',
+            type=int,
+            default=30,
+            help='The maximum number of days that reports should be allowed to exist. '
+            'Reports older than this amount of days will be deleted. Set to -1 to let '
+            'reports be kept indefinitely.'
+        )
+
+    def configure(
+        self,
+        connection_string: str,
+        container_name: str,
+        retention_days: int,
+    ):
+        '''
+        Configure the container client and other necessary data, such as the max cleanup time.
+
+        If a container with that name does not exist, it will be created.
+
+        Args:
+            connection_string: The full connection string to the storage account
+            container_name: The name of the container that should be used to store all images
+            retention_days: The maximum number of days a report should be allowed to have before
+                being cleaned up
+        '''
+        self._container_client = client = ContainerClient.from_connection_string(
+            connection_string,
+            container_name=container_name
+        )
+
+        self.max_retention_days = retention_days
+
+        if not client.exists():
+            client.create_container()
+
+    @property
+    def container_client(self) -> ContainerClient:
+        '''
+        An instance of the container client, casting it to ContainerClient.
+
+        This should be used exclusively from the abstract methods from BaseStorage
+
+        Returns:
+            The container client
+        '''
+
+        return cast(ContainerClient, self._container_client)
+
+    def quilla_configure(self, args: Namespace):
+        '''
+        Configures the plugin to run
+
+        Args:
+            args: A namespace generated by parsing the args from the CLI
+        '''
+        if args.connection_string is None:
+            return
+
+        self.configure(
+            args.connection_string,
+            args.container_name,
+            args.retention_days,
+        )
+
+    @property
+    def is_enabled(self) -> bool:
+        return self._container_client is not None
+
+    def find_image_by_baseline(self, baseline_id: str) -> bytes:
+        blob = self.container_client.get_blob_client(f'baselines/{baseline_id}.png')
+
+        if not blob.exists():
+            return b''
+
+        downloader = blob.download_blob()
+
+        blob_data = downloader.readall()
+
+        return blob_data
+
+    def store_baseline_image(self, run_id: str, baseline_id: str, baseline: bytes) -> str:
+        blob = self.container_client.get_blob_client(f'baselines/{baseline_id}.png')
+        snapshot = self.container_client.get_blob_client(
+            f'baselines/snapshots/{run_id}/{baseline_id}.png'
+        )
+
+        blob.upload_blob(baseline)
+        snapshot.upload_blob(baseline)
+
+        return blob.url
+
+    def store_treatment_image(self, run_id: str, baseline_id: str, treatment: bytes) -> str:
+        blob = self.container_client.get_blob_client(f'runs/{run_id}/{baseline_id}_treatment.png')
+        blob.upload_blob(treatment)
+
+        return blob.url
+
+    def make_baseline_uri(self, run_id: str, baseline_id: str) -> str:
+        baseline_data = self.find_image_by_baseline(baseline_id)
+        blob = self.container_client.get_blob_client(f'runs/{run_id}/{baseline_id}.png')
+
+        blob.upload_blob(baseline_data)
+
+        return blob.url
+
+    def cleanup_reports(self):
+        blobs = self.container_client.list_blobs()
+        current_time = datetime.now()
+
+        for blob in filter(lambda x: 'runs' in x.name, blobs):
+            time_created: datetime = datetime.fromtimestamp(blob.creation_time.timestamp())
+
+            delta = current_time - time_created
+            if self.max_retention_days > -1 and delta.days > self.max_retention_days:
+                blob_client = self.container_client.get_blob_client(blob)
+                blob_client.delete_blob()