Cache task groups and results (#112)

Fixes #86

mozci/task.py

@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Dict, List
 
+import adr
 import requests
 from adr.util import memoized_property
 from loguru import logger
@@ -184,9 +185,16 @@ class TestTask(Task):
             ]
 
     def _load_errorsummary(self):
+        # XXX: How or where should we invalidate the data?
+        data = adr.config.cache.get(self.id)
+        if data:
+            self._errors = data["errors"]
+            self._groups = data["groups"]
+            self._results = data["results"]
+            return None
         # This may clobber the values that were populated by ActiveData, but
         # since the artifact is already downloaded, parsed and we need to
-        # iterate over it anyway.. It doesn't really hurt and simplifies some
+        # iterate over it anyway. It doesn't really hurt and simplifies some
         # logic. It also ensures we don't attempt to load the errorsummary more
         # than once.
         self._groups = []
@@ -214,6 +222,19 @@ class TestTask(Task):
                 self._errors.append(line["message"])
 
         self.__post_init__()
+        # Only store data if there's something to store
+        if self._errors or self._groups or self._results:
+            logger.debug("Storing {} in the cache".format(self.id))
+            # cachy's put() overwrites the value in the cache; add() would only add if its empty
+            adr.config.cache.put(
+                self.id,
+                {
+                    "errors": self._errors,
+                    "groups": self._groups,
+                    "results": self._results,
+                },
+                adr.config._config["cache"]["retention"],
+            )
 
     @property
     def groups(self):

tests/test_integration.py

@@ -3,15 +3,36 @@ import os
 from argparse import Namespace
 
 import pytest
+from adr.configuration import Configuration
 from adr.query import run_query
 
 from mozci import task
+from mozci.push import Push
 
 pytestmark = pytest.mark.skipif(
     os.environ.get("TRAVIS_EVENT_TYPE") != "cron", reason="Not run by a cron task"
 )
 
 
+@pytest.fixture
+def adr_config(tmp_path):
+    config_file = tmp_path / "config.toml"
+    text = (
+        """
+[adr]
+verbose = true
+[adr.cache]
+default = "file"
+retention = 1000
+[adr.cache.stores]
+file = { driver = "file", path = "%s" }
+"""
+        % tmp_path
+    )
+    config_file.write_text(text)
+    return Configuration(path=config_file)
+
+
 def test_missing_manifests():
     """
     Ensure all suites (except a blacklist) are generating manifest information.
@@ -89,10 +110,36 @@ def test_good_result_manifests():
     """
     result = run_query("test_all_result_groups", Namespace())
 
-    for group, count in result["data"]:
+    for group, _ in result["data"]:
         if group is None:
             continue
 
         assert (
             not task.is_bad_group("x", group) and "\\" not in group
         ), f"{group} group is bad!"
+
+
+def test_caching_of_tasks(adr_config):
+    # Once we reach Nov. 23rd, 2020 the test should be updtated with a more recent push and task ID
+    TASK_ID = "WGNh9Xd8RmSG_170-0-mkQ"
+    # Making sure there's nothing left in the cache
+    assert adr_config.cache.get(TASK_ID) is None
+    # Push from Nov. 22nd, 2019
+    push = Push("6e87f52e6eebdf777f975baa407d5c22e9756e80", branch="mozilla-beta")
+    tasks = push.tasks
+    found_task = False
+    for t in tasks:
+        if t.id == TASK_ID:
+            task = adr_config.cache.get(TASK_ID)
+            assert task is None
+            # Calling one of the three properties will call _load_error_summary
+            # and cache the data
+            t.results
+            task = adr_config.cache.get(TASK_ID)
+            assert task["groups"]
+            assert not task["errors"]
+            assert not task["results"]
+            found_task = True
+            break
+
+    assert found_task