Implement the concept of data sources

.isort.cfg

@@ -2,4 +2,4 @@
 multi_line_output=3
 include_trailing_comma=True
 line_length=88
-known_third_party = adr,appdirs,boto3,botocore,cachy,loguru,pytest,requests,responses,taskcluster,taskcluster_urls,tomlkit,urllib3,yaml,zstandard
+known_third_party = adr,appdirs,boto3,botocore,cachy,loguru,pytest,requests,responses,taskcluster,taskcluster_urls,tomlkit,urllib3,voluptuous,yaml,zstandard

mozci/configuration.py

@@ -98,6 +98,7 @@ class CustomCacheManager(CacheManager):
 class Configuration(Mapping):
     DEFAULT_CONFIG_PATH = Path(user_config_dir("mozci")) / "config.toml"
     DEFAULTS = {
+        "data_sources": [],
         "cache": {"retention": 1440},  # minutes
         "verbose": False,
     }

mozci/data/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from mozci import config
+from mozci.data.base import DataHandler
+
+handler = DataHandler(*config.data_sources)

mozci/data/base.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+
+from abc import ABC, abstractproperty
+from typing import Any, Dict, Tuple
+
+from mozci.data.contract import all_contracts
+
+
+class DataSource(ABC):
+    def __init__(self) -> None:
+        missing = [
+            f"run_{c}"
+            for c in self.supported_contracts
+            if not hasattr(self, f"run_{c}")
+        ]
+        if missing:
+            missing_str = "  \n".join(missing)
+            raise Exception(
+                f"{self.__class__.__name__} must define the following methods:\n{missing_str}"
+            )
+
+    @abstractproperty
+    def name(self) -> str:
+        pass
+
+    @abstractproperty
+    def supported_contracts(self) -> Tuple[str, ...]:
+        pass
+
+    def get(self, name: str, **kwargs: Any) -> Dict[Any, Any]:
+        fn = getattr(self, f"run_{name}")
+        return fn(**kwargs)
+
+
+class DataHandler:
+    ALL_SOURCES: Dict[str, DataSource] = {}
+
+    def __init__(self, *sources: str) -> None:
+        self.sources = [self.ALL_SOURCES[sname] for sname in sources]
+
+    def get(self, name: str, **context: Any) -> Dict[Any, Any]:
+        """Given a contract, find the first registered source that supports it
+        run it and return the results.
+
+        Args:
+            name (str): Name of the contract to run.
+            context (dict): Context to pass into the contract as defined by `Contract.schema_in`.
+
+        Returns:
+            dict: The output of the contract as defined by `Contract.schema_out`.
+        """
+        if name not in all_contracts:
+            raise Exception(f"Contract {name} does not exist!")
+
+        # Validate input.
+        contract = all_contracts[name]
+        contract.validate_in(context)
+
+        source = None
+        for src in self.sources:
+            if name in src.supported_contracts:
+                source = src
+                break
+        else:
+            raise Exception(f"No registered sources support {name}!")
+
+        result = source.get(name, **context)
+
+        # Validate output.
+        contract.validate_out(result)
+        return result

mozci/data/contract.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+from dataclasses import dataclass
+from typing import Tuple
+
+from voluptuous import Schema
+
+
+@dataclass
+class Contract:
+    name: str
+    validate_in: Schema
+    validate_out: Schema
+
+
+_contracts: Tuple[Contract] = (
+    Contract(name="placeholder", validate_in=Schema({}), validate_out=Schema({}),),
+)
+
+
+all_contracts = {c.name: c for c in _contracts}

poetry.lock

@@ -177,6 +177,23 @@ optional = false
 python-versions = "*"
 version = "3.0.12"
 
+[[package]]
+category = "main"
+description = "the modular source code checker: pep8 pyflakes and co"
+name = "flake8"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
+version = "3.8.3"
+
+[package.dependencies]
+mccabe = ">=0.6.0,<0.7.0"
+pycodestyle = ">=2.6.0a1,<2.7.0"
+pyflakes = ">=2.2.0,<2.3.0"
+
+[package.dependencies.importlib-metadata]
+python = "<3.8"
+version = "*"
+
 [[package]]
 category = "dev"
 description = "File identification library for Python"
@@ -300,6 +317,14 @@ optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
 version = "1.1.1"
 
+[[package]]
+category = "main"
+description = "McCabe checker, plugin for flake8"
+name = "mccabe"
+optional = false
+python-versions = "*"
+version = "0.6.1"
+
 [[package]]
 category = "main"
 description = "Library for Hawk HTTP authorization"
@@ -383,6 +408,22 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 version = "1.9.0"
 
+[[package]]
+category = "main"
+description = "Python style guide checker"
+name = "pycodestyle"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "2.6.0"
+
+[[package]]
+category = "main"
+description = "passive checker of Python programs"
+name = "pyflakes"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "2.2.0"
+
 [[package]]
 category = "main"
 description = "Pygments is a syntax highlighting package written in Python."
@@ -754,6 +795,22 @@ version = ">=0.12,<2"
 docs = ["sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)", "proselint (>=0.10.2)"]
 testing = ["pytest (>=4)", "coverage (>=5)", "coverage-enable-subprocess (>=1)", "pytest-xdist (>=1.31.0)", "pytest-mock (>=2)", "pytest-env (>=0.6.2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "pytest-freezegun (>=0.4.1)", "flaky (>=3)", "packaging (>=20.0)", "xonsh (>=0.9.16)"]
 
+[[package]]
+category = "main"
+description = "# Voluptuous is a Python data validation library"
+name = "voluptuous"
+optional = false
+python-versions = "*"
+version = "0.11.7"
+
+[[package]]
+category = "dev"
+description = "Measures the displayed width of unicode strings in a terminal"
+name = "wcwidth"
+optional = false
+python-versions = "*"
+version = "0.2.5"
+
 [[package]]
 category = "main"
 description = "A small Python utility to set file creation time on Windows"
@@ -857,6 +914,10 @@ filelock = [
     {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"},
     {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"},
 ]
+flake8 = [
+    {file = "flake8-3.8.3-py2.py3-none-any.whl", hash = "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c"},
+    {file = "flake8-3.8.3.tar.gz", hash = "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208"},
+]
 identify = [
     {file = "identify-1.4.21-py2.py3-none-any.whl", hash = "sha256:dac33eff90d57164e289fb20bf4e131baef080947ee9bf45efcd0da8d19064bf"},
     {file = "identify-1.4.21.tar.gz", hash = "sha256:c4d07f2b979e3931894170a9e0d4b8281e6905ea6d018c326f7ffefaf20db680"},
@@ -931,6 +992,10 @@ markupsafe = [
     {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"},
     {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
 ]
+mccabe = [
+    {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
+    {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
+]
 mohawk = [
     {file = "mohawk-1.1.0-py3-none-any.whl", hash = "sha256:3ed296a30453d0b724679e0fd41e4e940497f8e461a9a9c3b7f36e43bab0fa09"},
     {file = "mohawk-1.1.0.tar.gz", hash = "sha256:d2a0e3ab10a209cc79e95e28f2dd54bd4a73fd1998ffe27b7ba0f962b6be9723"},
@@ -958,6 +1023,14 @@ py = [
     {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"},
     {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"},
 ]
+pycodestyle = [
+    {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"},
+    {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"},
+]
+pyflakes = [
+    {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"},
+    {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"},
+]
 pygments = [
     {file = "Pygments-2.6.1-py3-none-any.whl", hash = "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324"},
     {file = "Pygments-2.6.1.tar.gz", hash = "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44"},
@@ -1083,6 +1156,13 @@ virtualenv = [
     {file = "virtualenv-20.0.26-py2.py3-none-any.whl", hash = "sha256:c11a475400e98450403c0364eb3a2d25d42f71cf1493da64390487b666de4324"},
     {file = "virtualenv-20.0.26.tar.gz", hash = "sha256:e10cc66f40cbda459720dfe1d334c4dc15add0d80f09108224f171006a97a172"},
 ]
+voluptuous = [
+    {file = "voluptuous-0.11.7.tar.gz", hash = "sha256:2abc341dbc740c5e2302c7f9b8e2e243194fb4772585b991931cb5b22e9bf456"},
+]
+wcwidth = [
+    {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"},
+    {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
+]
 win32-setctime = [
     {file = "win32_setctime-1.0.1-py3-none-any.whl", hash = "sha256:568fd636c68350bcc54755213fe01966fe0a6c90b386c0776425944a0382abef"},
     {file = "win32_setctime-1.0.1.tar.gz", hash = "sha256:b47e5023ec7f0b4962950902b15bc56464a380d869f59d27dbf9ab423b23e8f9"},

pyproject.toml

@@ -20,6 +20,8 @@ zstandard = {version = "^0.14.0", optional = true}
 python3-memcached = {version = "^1.51", optional = true}
 redis = {version = "^3.5.3", optional = true}
 requests = "^2.24.0"
+voluptuous = "^0.11.7"
+flake8 = "^3.8.3"
 
 [tool.poetry.dev-dependencies]
 pre-commit = "^2.6"

tests/test_data.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+import pytest
+from voluptuous import MultipleInvalid, Required, Schema
+
+from mozci import data
+from mozci.data.base import DataHandler, DataSource
+from mozci.data.contract import Contract
+
+FAKE_CONTRACTS = (
+    Contract(
+        name="foo",
+        validate_in=Schema({Required("label"): str}),
+        validate_out=Schema({Required("count"): int}),
+    ),
+    Contract(
+        name="bar",
+        validate_in=Schema({Required("desc"): str}),
+        validate_out=Schema({Required("amount"): int}),
+    ),
+    Contract(
+        name="baz",
+        validate_in=Schema({Required("id"): str}),
+        validate_out=Schema({Required("sum"): int}),
+    ),
+)
+
+
+class FakeSource(DataSource):
+    name = "fake"
+    supported_contracts = ["foo", "bar"]
+
+    def run_foo(self, **context):
+        return {"count": "1"}
+
+    def run_bar(self, **context):
+        return {"amount": 1}
+
+
+class InvalidSource(DataSource):
+    name = "invalid"
+    supported_contracts = ["foo"]
+
+
+def test_data_handler(monkeypatch):
+    with pytest.raises(Exception):
+        DataHandler("nonexistent")
+
+    monkeypatch.setattr(data.base, "all_contracts", {c.name: c for c in FAKE_CONTRACTS})
+    monkeypatch.setattr(DataHandler, "ALL_SOURCES", {"fake": FakeSource()})
+    handler = DataHandler("fake")
+
+    with pytest.raises(Exception):
+        handler.get("baz")
+
+    with pytest.raises(Exception):
+        handler.get("fleem")
+
+    with pytest.raises(MultipleInvalid):
+        handler.get("foo")
+
+    with pytest.raises(MultipleInvalid):
+        handler.get("foo", label="foo")
+
+    assert handler.get("bar", desc="tada") == {"amount": 1}
+
+
+def test_data_source():
+    with pytest.raises(Exception):
+        InvalidSource()
+
+    source = FakeSource()
+
+    with pytest.raises(Exception):
+        source.get("baz")
+
+    assert source.get("foo") == {"count": "1"}
+    assert source.get("bar") == {"amount": 1}

tox.ini

@@ -4,7 +4,6 @@ envlist = py37,pre-commit
 
 [testenv]
 envdir = {toxworkdir}/env
-deps = poetry
 passenv = TRAVIS_EVENT_TYPE
 setenv =
     MOZCI_CONFIG_PATH = tests/config.toml