Bug 1643469 - Swap NotebookLogger for MachLogger. r=sparky

Differential Revision: https://phabricator.services.mozilla.com/D81156

python/mozperftest/mozperftest/metrics/common.py

@@ -135,7 +135,13 @@ class MetricsStorage(object):
                 "file_groups": {data_type: data_info["files"]},
             }
 
-            ptnb = PerftestETL(config["file_groups"], config, data_info["transformer"])
+            ptnb = PerftestETL(
+                file_groups=config["file_groups"],
+                config=config,
+                prefix=self.prefix,
+                logger=self.logger,
+                custom_transform=data_info["transformer"],
+            )
             r = ptnb.process(**data_info["options"])
             self.stddata[data_type] = r["data"]
 

python/mozperftest/mozperftest/metrics/notebook/logger.py

@@ -1,35 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import logging
-
-logging.basicConfig(level=logging.INFO)
-logger = None
-
-
-class NotebookLogger(object):
-    """Simple logger for perftest-notebook.
-    """
-
-    debug = False
-
-    def __init__(self, name="perftest-notebook"):
-        self._logger = logger
-        if not self._logger:
-            self._logger = logging.getLogger(name)
-
-    def debug(self, msg):
-        if self.debug:
-            self._logger.info(msg)
-
-    def info(self, msg):
-        self._logger.info(msg)
-
-    def warning(self, msg):
-        self._logger.warning(msg)
-
-    def error(self, msg):
-        self._logger.error(msg)
-
-    def critical(self, msg):
-        self._logger.critical(msg)

python/mozperftest/mozperftest/metrics/notebook/perftestetl.py

@@ -9,16 +9,21 @@ from collections import OrderedDict
 
 from .transformer import Transformer, SimplePerfherderTransformer
 from .constant import Constant
-from .logger import NotebookLogger
-
-logger = NotebookLogger()
 
 
 class PerftestETL(object):
     """Controller class for the PerftestETL."""
 
-    def __init__(self, file_groups, config, custom_transform=None, sort_files=False):
-        """Initializes PerftestNotebook.
+    def __init__(
+        self,
+        file_groups,
+        config,
+        prefix,
+        logger,
+        custom_transform=None,
+        sort_files=False,
+    ):
+        """Initializes PerftestETL.
 
         :param dict file_groups: A dict of file groupings. The value
             of each of the dict entries is the name of the data that
@@ -30,6 +35,8 @@ class PerftestETL(object):
         self.config = config
         self.sort_files = sort_files
         self.const = Constant()
+        self.prefix = prefix
+        self.logger = logger
 
         # Gather the available transformers
         tfms_dict = self.const.predefined_transformers
@@ -45,13 +52,21 @@ class PerftestETL(object):
         if custom_transform:
             tfm_cls = tfms_dict.get(custom_transform)
             if tfm_cls:
-                self.transformer = Transformer(files=[], custom_transformer=tfm_cls())
-                logger.info(f"Found {custom_transform} transformer")
+                self.transformer = Transformer(
+                    files=[],
+                    custom_transformer=tfm_cls(),
+                    logger=self.logger,
+                    prefix=self.prefix,
+                )
+                self.logger.info(f"Found {custom_transform} transformer", self.prefix)
             else:
                 raise Exception(f"Could not get a {custom_transform} transformer.")
         else:
             self.transformer = Transformer(
-                files=[], custom_transformer=SimplePerfherderTransformer()
+                files=[],
+                custom_transformer=SimplePerfherderTransformer(),
+                logger=self.logger,
+                prefix=self.prefix,
             )
 
     def parse_file_grouping(self, file_grouping):

python/mozperftest/mozperftest/metrics/notebook/perftestnotebook.py

@@ -6,18 +6,19 @@ import webbrowser
 from http.server import BaseHTTPRequestHandler, HTTPServer
 
 from .constant import Constant
-from .logger import logger
 
 
 class PerftestNotebook(object):
     """Controller class for PerftestNotebook."""
 
-    def __init__(self, data):
+    def __init__(self, data, logger, prefix):
         """Initialize the PerftestNotebook.
 
         :param dict data: Standardized data, post-transformation.
         """
         self.data = data
+        self.logger = logger
+        self.prefix = prefix
         self.const = Constant()
 
     def get_notebook_section(self, func):
@@ -27,7 +28,9 @@ class PerftestNotebook(object):
         """
         template_path = self.const.here / "notebook-sections" / func
         if not template_path.exists():
-            logger.warning(f"Could not find the notebook-section called {func}")
+            self.logger.warning(
+                f"Could not find the notebook-section called {func}", self.prefix
+            )
             return ""
         with template_path.open() as f:
             return f.read()

python/mozperftest/mozperftest/metrics/notebook/transformer.py

@@ -8,7 +8,6 @@ import sys
 import pathlib
 
 from jsonschema import validate
-from .logger import NotebookLogger
 from mozperftest.metrics.exceptions import (
     NotebookInvalidTransformError,
     NotebookInvalidPathError,
@@ -16,14 +15,12 @@ from mozperftest.metrics.exceptions import (
 )
 from mozperftest.runner import HERE
 
-logger = NotebookLogger()
-
 
 class Transformer(object):
     """Abstract class for data transformers.
     """
 
-    def __init__(self, files=None, custom_transformer=None):
+    def __init__(self, files=None, custom_transformer=None, logger=None, prefix=None):
         """Initialize the transformer with files.
 
         :param list files: A list of files containing data to transform.
@@ -31,6 +28,8 @@ class Transformer(object):
             Must implement `transform` and `merge` methods.
         """
         self._files = files
+        self.logger = logger
+        self.prefix = prefix
 
         if custom_transformer:
             valid = (
@@ -57,7 +56,9 @@ class Transformer(object):
     @files.setter
     def files(self, val):
         if not isinstance(val, list):
-            logger.warning("`files` must be a list, got %s" % type(val))
+            self.logger.warning(
+                "`files` must be a list, got %s" % type(val), self.prefix
+            )
             return
         self._files = val
 
@@ -97,8 +98,10 @@ class Transformer(object):
                 else:
                     data = self.open_data(file)
             except Exception as e:
-                logger.warning("Failed to open file %s, skipping" % file)
-                logger.warning("%s %s" % (e.__class__.__name__, e))
+                self.logger.warning(
+                    "Failed to open file %s, skipping" % file, self.prefix
+                )
+                self.logger.warning("%s %s" % (e.__class__.__name__, e), self.prefix)
 
             # Transform data
             try:
@@ -110,8 +113,10 @@ class Transformer(object):
                         ele.update({"file": file})
                 trfmdata.extend(data)
             except Exception as e:
-                logger.warning("Failed to transform file %s, skipping" % file)
-                logger.warning("%s %s" % (e.__class__.__name__, e))
+                self.logger.warning(
+                    "Failed to transform file %s, skipping" % file, self.prefix
+                )
+                self.logger.warning("%s %s" % (e.__class__.__name__, e), self.prefix)
 
         merged = self._custom_transformer.merge(trfmdata)
 

python/mozperftest/mozperftest/metrics/notebookupload.py

@@ -63,7 +63,9 @@ class Notebook(Layer):
                     data_to_post.append(r)
                 elif self.get_arg("analyze-strings"):
                     data_to_post.append(r)
-        self.ptnb = PerftestNotebook(data_to_post)
+        self.ptnb = PerftestNotebook(
+            data=data_to_post, logger=metadata, prefix=self.get_arg("prefix")
+        )
         self.ptnb.post_to_iodide(analysis)
 
         return metadata

python/mozperftest/mozperftest/tests/conftest.py

@@ -3,7 +3,7 @@ import os
 import pathlib
 import pytest
 from mozperftest.metrics.notebook.perftestetl import PerftestETL
-from mozperftest.tests.support import temp_dir, HERE
+from mozperftest.tests.support import get_running_env, temp_dir, HERE
 from mozperftest.metrics.notebook.perftestnotebook import PerftestNotebook
 
 
@@ -94,19 +94,33 @@ def files(data):
 @pytest.fixture(scope="session", autouse=True)
 def ptetls(files):
     resources, dirs, output = files["resources"], files["dirs"], files["output"]
+    _, metadata, _ = get_running_env()
     config = {"output": output}
     file_group_list = {"group_1": list(resources.values())}
     file_group_str = {"group_1": dirs["resources"].resolve().as_posix()}
 
     yield {
-        "ptetl_list": PerftestETL(file_group_list, config, sort_files=True),
-        "ptetl_str": PerftestETL(file_group_str, config, sort_files=True),
+        "ptetl_list": PerftestETL(
+            file_groups=file_group_list,
+            config=config,
+            prefix="PerftestETL",
+            logger=metadata,
+            sort_files=True,
+        ),
+        "ptetl_str": PerftestETL(
+            file_groups=file_group_str,
+            config=config,
+            prefix="PerftestETL",
+            logger=metadata,
+            sort_files=True,
+        ),
     }
 
 
 @pytest.fixture(scope="session", autouse=True)
 def ptnb(standarized_data):
-    return PerftestNotebook(standarized_data)
+    _, metadata, _ = get_running_env()
+    return PerftestNotebook(standarized_data, metadata, "PerftestNotebook")
 
 
 @pytest.fixture(scope="function", autouse=True)

python/mozperftest/mozperftest/tests/test_notebookupload.py

@@ -43,7 +43,7 @@ def test_notebookupload_with_filter(notebook, no_filter):
 
     if no_filter:
         args, kwargs = notebook.call_args_list[0]
-        assert type(args[0][0]["data"][0]["value"]) == str
+        assert type(kwargs["data"][0]["data"][0]["value"]) == str
     else:
         for call in notebook.call_args_list:
             args, kwargs = call

python/mozperftest/mozperftest/tests/test_perftestnotebook.py

@@ -19,11 +19,7 @@ def test_get_notebook_section(ptnb):
 
 def test_get_notebook_section_unknown_analysis(ptnb):
     func = "unknown"
-    with mock.patch("mozperftest.metrics.notebook.perftestnotebook.logger") as logger:
-        assert ptnb.get_notebook_section(func) == ""
-        logger.assert_has_calls(
-            mock.call.warning("Could not find the notebook-section called unknown")
-        )
+    assert ptnb.get_notebook_section(func) == ""
 
 
 @pytest.mark.parametrize("analysis", [["scatterplot"], None])

python/mozperftest/mozperftest/tests/test_transformer.py

@@ -1,7 +1,7 @@
 import mozunit
 import pytest
 from jsonschema import ValidationError
-from mozperftest.tests.support import HERE
+from mozperftest.tests.support import get_running_env, HERE
 from mozperftest.metrics.notebook.transformer import Transformer, get_transformers
 from mozperftest.tests.data.perftestetl_plugin import (
     test_transformer_perftestetl_plugin_1,
@@ -13,6 +13,9 @@ from mozperftest.metrics.exceptions import (
     NotebookDuplicateTransformsError,
 )
 
+_, metadata, _ = get_running_env()
+prefix = "PerftestNotebook"
+
 
 def test_init_failure():
     class TempClass(object):
@@ -25,25 +28,25 @@ def test_init_failure():
 
 def test_files_getter(files):
     files = files["resources"]
-    assert files == Transformer(files).files
+    assert files == Transformer(files, logger=metadata, prefix=prefix).files
 
 
 def test_files_setter(files):
     files = files["resources"]
     files = list(files.values())
-    tfm = Transformer()
+    tfm = Transformer(logger=metadata, prefix=prefix)
     tfm.files = files
     assert files == tfm.files
 
 
 def test_files_setter_failure():
-    tfm = Transformer()
+    tfm = Transformer(logger=metadata, prefix=prefix)
     tfm.files = "fail"
     assert not tfm.files
 
 
 def test_open_data(data, files):
-    tfm = Transformer()
+    tfm = Transformer(logger=metadata, prefix=prefix)
 
     files = files["resources"]
     json_1 = files["file_1"]
@@ -73,7 +76,7 @@ def test_jsonschema_valitate_failure(files):
     file_1 = files["file_1"]
     file_2 = files["file_2"]
 
-    tfm = Transformer([], BadTransformer())
+    tfm = Transformer([], BadTransformer(), logger=metadata, prefix=prefix)
     tfm.files = [file_1, file_2]
     with pytest.raises(ValidationError):
         tfm.process("name")