Bug 1295536 - Validate that perfherder values are within acceptable ranges

Especially make sure that we have no "infinite" values, as those can
cause exceptions.

schemas/performance-artifact.json

@@ -18,7 +18,9 @@
                 "value": {
                     "description": "Summary value for subtest",
                     "title": "Subtest value",
-                    "type": "number"
+                    "type": "number",
+                    "minimum": -1000000000000.0,
+                    "maximum": 1000000000000.0
                 },
                 "lowerIsBetter": {
                     "description": "Whether lower values are better for subtest",
@@ -33,22 +35,30 @@
                 "alertThreshold": {
                     "description": "% change threshold before alerting",
                     "title": "Alert threshold",
-                    "type": "number"
+                    "type": "number",
+                    "minimum": 0.0,
+                    "maximum": 1000.0
                 },
                 "minBackWindow": {
                     "description": "Minimum back window to use for alerting",
                     "title": "Minimum back window",
-                    "type": "number"
+                    "type": "number",
+                    "minimum": 1,
+                    "maximum": 255
                 },
                 "maxBackWindow": {
                     "description": "Maximum back window to use for alerting",
                     "title": "Maximum back window",
-                    "type": "number"
+                    "type": "number",
+                    "minimum": 1,
+                    "maximum": 255
                 },
                 "foreWindow": {
                     "description": "Fore window to use for alerting",
                     "title": "Fore window",
-                    "type": "number"
+                    "type": "number",
+                    "minimum": 1,
+                    "maximum": 255
                 }
             },
             "required": [
@@ -80,7 +90,9 @@
                 },
                 "value": {
                     "title": "Suite value",
-                    "type": "number"
+                    "type": "number",
+                    "minimum": -1000000000000.0,
+                    "maximum": 1000000000000.0
                 },
                 "lowerIsBetter": {
                     "description": "Whether lower values are better for suite",
@@ -95,22 +107,30 @@
                 "alertThreshold": {
                     "description": "% change threshold before alerting",
                     "title": "Alert threshold",
-                    "type": "number"
+                    "type": "number",
+                    "minimum": 0.0,
+                    "maximum": 1000.0
                 },
                 "minBackWindow": {
                     "description": "Minimum back window to use for alerting",
                     "title": "Minimum back window",
-                    "type": "number"
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 255
                 },
                 "maxBackWindow": {
                     "description": "Maximum back window to use for alerting",
                     "title": "Maximum back window",
-                    "type": "number"
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 255
                 },
                 "foreWindow": {
                     "description": "Fore window to use for alerting",
                     "title": "Fore window",
-                    "type": "number"
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 255
                 }
             },
             "required": [

tests/etl/test_perf_schema.py

@@ -0,0 +1,31 @@
+import json
+
+import jsonschema
+import pytest
+
+
+@pytest.mark.parametrize(('suite_value', 'test_value', 'expected_fail'),
+                         [({}, {}, True),
+                          ({'value': 1234}, {}, True),
+                          ({}, {'value': 1234}, False),
+                          ({'value': 1234}, {'value': 1234}, False),
+                          ({'value': float('inf')}, {}, True),
+                          ({}, {'value': float('inf')}, True)])
+def test_perf_schema(suite_value, test_value, expected_fail):
+    perf_schema = json.load(open('schemas/performance-artifact.json'))
+    datum = {
+        "framework": {"name": "talos"}, "suites": [{
+            "name": "basic_compositor_video",
+            "subtests": [{
+                "name": "240p.120fps.mp4_scale_fullscreen_startup"
+            }]
+        }]
+    }
+    datum['suites'][0].update(suite_value)
+    datum['suites'][0]['subtests'][0].update(test_value)
+    print datum
+    if expected_fail:
+        with pytest.raises(jsonschema.ValidationError):
+            jsonschema.validate(datum, perf_schema)
+    else:
+        jsonschema.validate(datum, perf_schema)