Bug 1640951 - Implement a jsonschema for results coming from transformer merge functions. r=sparky

Differential Revision: https://phabricator.services.mozilla.com/D78622

python/mozperftest/mozperftest/metrics/notebook/transformer.py

@@ -5,13 +5,16 @@ import json
 import importlib.util
 import inspect
 import sys
+import pathlib
 
+from jsonschema import validate
 from .logger import NotebookLogger
 from mozperftest.metrics.exceptions import (
     NotebookInvalidTransformError,
     NotebookInvalidPathError,
     NotebookDuplicateTransformsError,
 )
+from mozperftest.runner import HERE
 
 logger = NotebookLogger()
 
@@ -44,6 +47,9 @@ class Transformer(object):
 
         self._custom_transformer = custom_transformer
 
+        with pathlib.Path(HERE, "schemas", "transformer_schema.json").open() as f:
+            self.schema = json.load(f)
+
     @property
     def files(self):
         return self._files
@@ -115,6 +121,7 @@ class Transformer(object):
             for e in merged:
                 e["name"] = name
 
+        validate(instance=merged, schema=self.schema)
         return merged
 
 

python/mozperftest/mozperftest/metrics/notebook/transforms/logcattime.py

@@ -102,10 +102,11 @@ class LogCatTimeTransformer:
             second_ts=kwargs.get("second-timestamp"),
             processor=kwargs.get("processor"),
         )
+        subtest = kwargs.get("transform-subtest-name")
         return [
             {
                 "data": [{"value": val, "xaxis": c} for c, val in enumerate(alltimes)],
-                "subtest": kwargs.get("transform-subtest-name", "logcat-metric"),
+                "subtest": subtest if subtest else "logcat-metric",
             }
         ]
 

python/mozperftest/mozperftest/schemas/transformer_schema.json

@@ -0,0 +1,55 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "definitions": {
+        "data": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "file": {
+                        "type": "string"
+                    },
+                    "value": {},
+                    "xaxis": {
+                        "type": "number"
+                    }
+                },
+                "required": [
+                    "file",
+                    "value",
+                    "xaxis"
+                ]
+            }
+        },
+        "dict": {
+            "type": "object",
+            "properties": {
+                "data": {
+                    "$ref": "#/definitions/data"
+                },
+                "name": {
+                    "type": "string"
+                },
+                "subtest": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "data",
+                "name",
+                "subtest"
+            ]
+        }
+    },
+    "oneOf": [
+        {
+            "$ref": "#/definitions/dict"
+        },
+        {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/dict"
+            }
+        }
+    ]
+}

python/mozperftest/mozperftest/tests/test_transformer.py

@@ -1,5 +1,6 @@
 import mozunit
 import pytest
+from jsonschema import ValidationError
 from mozperftest.tests.support import HERE
 from mozperftest.metrics.notebook.transformer import Transformer, get_transformers
 from mozperftest.tests.data.perftestetl_plugin import (
@@ -60,6 +61,24 @@ def test_open_data(data, files):
         tfm.open_data("fail")
 
 
+def test_jsonschema_valitate_failure(files):
+    class BadTransformer:
+        def transform(self, data):
+            return {"bad data": "bad data"}
+
+        def merge(self, sde):
+            return {"bad data": "bad data"}
+
+    files = files["resources"]
+    file_1 = files["file_1"]
+    file_2 = files["file_2"]
+
+    tfm = Transformer([], BadTransformer())
+    tfm.files = [file_1, file_2]
+    with pytest.raises(ValidationError):
+        tfm.process("name")
+
+
 def test_get_transformers():
     dirpath = HERE / "data" / "perftestetl_plugin"
     tfms = get_transformers(dirpath)

python/mozperftest/mozperftest/utils.py

@@ -42,6 +42,8 @@ def silence(layer=None):
     oldout, olderr = sys.stdout, sys.stderr
     try:
         sys.stdout, sys.stderr = StringIO(), StringIO()
+        sys.stdout.buffer = sys.stdout
+        sys.stderr.buffer = sys.stderr
         sys.stdout.fileno = sys.stderr.fileno = lambda: -1
         yield sys.stdout, sys.stderr
     finally: