Papertrail support (#115)

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
-known_first_party = code_review_bot,code_review_events,conftest
+known_first_party = code_review_bot,code_review_tools,code_review_events,conftest
 known_third_party = influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,requests,responses,setuptools,structlog,taskcluster,toml
 force_single_line = True
 default_section=FIRSTPARTY

.taskcluster.yml

@@ -45,8 +45,9 @@ tasks:
         command:
           - sh
           - -lxce
-          - "git clone --quiet ${repository} /src && cd /src/bot && git checkout ${head_rev} -b checks &&
-            pip install . && pip install -r requirements-dev.txt &&
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            cd /src/tools && pip install . &&
+            cd /src/bot && pip install . && pip install -r requirements-dev.txt &&
             cd /src && pre-commit run -a"
       metadata:
         name: "Code Review Bot checks: linting"
@@ -66,8 +67,9 @@ tasks:
           - sh
           - -lxce
           - "apk add git --quiet &&
-            git clone --quiet ${repository} /src && cd /src/bot && git checkout ${head_rev} -b checks &&
-            pip install . && pip install -r requirements-dev.txt &&
+            git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            cd /src/tools && pip install . &&
+            cd /src/bot && pip install . && pip install -r requirements-dev.txt &&
             pytest -v"
       metadata:
         name: "Code Review Bot checks: unit tests"
@@ -86,8 +88,9 @@ tasks:
         command:
           - sh
           - -lxce
-          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && cd /src/events &&
-            pip install -r requirements.txt && pip install . && pip install -r requirements-dev.txt &&
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            cd /src/tools && pip install . &&
+            cd /src/events && pip install . && pip install -r requirements-dev.txt &&
             pytest -v"
       metadata:
         name: "Code Review Events checks: unit tests"
@@ -147,7 +150,7 @@ tasks:
           - "${head_rev}"
           - --write
           - /bot.tar
-          - bot/Dockerfile
+          - bot/docker/Dockerfile
         artifacts:
           public/code-review-bot.tar:
             expires: {$fromNow: '2 weeks'}

bot/Dockerfile

@@ -1,6 +0,0 @@
-FROM python:3-alpine
-
-COPY bot /src
-RUN pip install /src
-
-CMD ["code-review-bot"]

bot/code_review_bot/__init__.py

@@ -9,7 +9,7 @@ import enum
 from code_review_bot.config import Publication
 from code_review_bot.config import settings
 from code_review_bot.stats import InfluxDb
-from code_review_bot.tools.taskcluster import TaskclusterConfig
+from code_review_tools.taskcluster import TaskclusterConfig
 
 CLANG_TIDY = "clang-tidy"
 CLANG_FORMAT = "clang-format"

bot/code_review_bot/cli.py

@@ -17,8 +17,8 @@ from code_review_bot import taskcluster
 from code_review_bot.config import settings
 from code_review_bot.report import get_reporters
 from code_review_bot.revisions import Revision
-from code_review_bot.tools.log import init_logger
 from code_review_bot.workflow import Workflow
+from code_review_tools.log import init_logger
 
 logger = structlog.get_logger(__name__)
 
@@ -57,7 +57,8 @@ def main():
     )
 
     init_logger(
-        config.PROJECT_NAME,
+        "bot",
+        channel=taskcluster.secrets.get("APP_CHANNEL", "dev"),
         PAPERTRAIL_HOST=taskcluster.secrets.get("PAPERTRAIL_HOST"),
         PAPERTRAIL_PORT=taskcluster.secrets.get("PAPERTRAIL_PORT"),
         SENTRY_DSN=taskcluster.secrets.get("SENTRY_DSN"),

bot/code_review_bot/revisions.py

@@ -16,7 +16,7 @@ from code_review_bot import stats
 from code_review_bot.config import REPO_GECKO_TRY
 from code_review_bot.config import REPO_NSS_TRY
 from code_review_bot.config import settings
-from code_review_bot.tools.taskcluster import create_blob_artifact
+from code_review_tools.taskcluster import create_blob_artifact
 
 logger = structlog.get_logger(__name__)
 

bot/code_review_bot/workflow.py

@@ -21,7 +21,7 @@ from code_review_bot.tasks.coverage import ZeroCoverageTask
 from code_review_bot.tasks.coverity import CoverityTask
 from code_review_bot.tasks.infer import InferTask
 from code_review_bot.tasks.lint import MozLintTask
-from code_review_bot.tools.taskcluster import TASKCLUSTER_DATE_FORMAT
+from code_review_tools.taskcluster import TASKCLUSTER_DATE_FORMAT
 
 logger = structlog.get_logger(__name__)
 

bot/docker/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3-alpine
+
+ADD tools /src/tools
+ADD bot /src/bot
+
+RUN /src/bot/docker/bootstrap.sh
+
+CMD ["code-review-bot"]

bot/docker/bootstrap.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Add build deps
+apk --update add --virtual build-dependencies gcc musl-dev libffi-dev openssl-dev
+
+# Install python dependencies
+cd /src/tools && python setup.py install
+cd /src/bot && python setup.py install
+
+# Cleanup
+apk del build-dependencies
+rm -rf /var/cache/apk/*

bot/requirements.txt

@@ -1,7 +1,5 @@
+-e ../tools #egg=code-review-tools
 influxdb==5.2.3
 libmozdata==0.1.59
-Logbook==1.5.2
 parsepatch==0.1.3
-structlog==19.1.0
-taskcluster==16.2.0
 toml==0.10.0

bot/tests/test_tools.py

@@ -5,7 +5,7 @@
 
 import pytest
 
-from code_review_bot.tools.taskcluster import TaskclusterConfig
+from code_review_tools.taskcluster import TaskclusterConfig
 
 
 def test_taskcluster_service():

events/code_review_events/cli.py

@@ -5,9 +5,9 @@ import tempfile
 
 import structlog
 from libmozevent import taskcluster_config
-from libmozevent.log import init_logger
 
 from code_review_events.workflow import Events
+from code_review_tools.log import init_logger
 
 logger = structlog.get_logger(__name__)
 
@@ -45,7 +45,8 @@ def main():
     )
 
     init_logger(
-        "code_review_events",
+        "events",
+        channel=taskcluster_config.secrets.get("APP_CHANNEL", "dev"),
         PAPERTRAIL_HOST=taskcluster_config.secrets.get("PAPERTRAIL_HOST"),
         PAPERTRAIL_PORT=taskcluster_config.secrets.get("PAPERTRAIL_PORT"),
         SENTRY_DSN=taskcluster_config.secrets.get("SENTRY_DSN"),

events/docker/Dockerfile

@@ -1,9 +1,11 @@
 FROM python:3-slim
 
-ADD events /src
+ADD tools /src/tools
+ADD events /src/events
 
-RUN cd /src && pip install --disable-pip-version-check --no-cache-dir -r requirements.txt && python setup.py install
+RUN cd /src/tools && python setup.py install
+RUN cd /src/events && python setup.py install
 
-RUN /src/docker/bootstrap.sh
+RUN /src/events/docker/bootstrap.sh
 
 CMD ["code-review-events"]

events/requirements.txt

@@ -1 +1,2 @@
+-e ../tools #egg=code-review-tools
 libmozevent==1.0.0

events/setup.py

@@ -11,8 +11,24 @@ here = os.path.dirname(__file__)
 
 
 def read_requirements(file_):
-    with open(os.path.join(here, file_)) as f:
-        return sorted(list(set(line.split("#")[0].strip() for line in f)))
+    lines = []
+    with open(file_) as f:
+        for line in f.readlines():
+            line = line.strip()
+            if (
+                line.startswith("-e ")
+                or line.startswith("http://")
+                or line.startswith("https://")
+            ):
+                extras = ""
+                if "[" in line:
+                    extras = "[" + line.split("[")[1].split("]")[0] + "]"
+                line = line.split("#")[1].split("egg=")[1] + extras
+            elif line == "" or line.startswith("#") or line.startswith("-"):
+                continue
+            line = line.split("#")[0].strip()
+            lines.append(line)
+    return sorted(list(set(lines)))
 
 
 with open(os.path.join(here, "VERSION")) as f:

tools/code_review_tools/log.py

@@ -33,7 +33,7 @@ def setup_papertrail(project_name, channel, PAPERTRAIL_HOST, PAPERTRAIL_PORT):
 
     # Setup papertrail
     papertrail = logbook.SyslogHandler(
-        application_name=f"mozilla/release-services/{channel}/{project_name}",
+        application_name=f"code-review/{channel}/{project_name}",
         address=(PAPERTRAIL_HOST, int(PAPERTRAIL_PORT)),
         level=logbook.INFO,
         format_string="{record.time} {record.channel}: {record.message}",

tools/requirements.txt

@@ -0,0 +1,4 @@
+Logbook==1.5.2
+structlog==19.1.0
+taskcluster==16.2.0
+raven==6.10.0

tools/setup.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import setuptools
+
+
+def read_requirements(file_):
+    lines = []
+    with open(file_) as f:
+        for line in f.readlines():
+            line = line.strip()
+            if (
+                line.startswith("-e ")
+                or line.startswith("http://")
+                or line.startswith("https://")
+            ):
+                extras = ""
+                if "[" in line:
+                    extras = "[" + line.split("[")[1].split("]")[0] + "]"
+                line = line.split("#")[1].split("egg=")[1] + extras
+            elif line == "" or line.startswith("#") or line.startswith("-"):
+                continue
+            line = line.split("#")[0].strip()
+            lines.append(line)
+    return sorted(list(set(lines)))
+
+
+setuptools.setup(
+    name="code-review-tools",
+    version="0.1.0",
+    description="Support tools for Mozilla code review",
+    author="Mozilla Release Management",
+    author_email="release-mgmt-analysis@mozilla.com",
+    install_requires=read_requirements("requirements.txt"),
+    packages=setuptools.find_packages(),
+    include_package_data=True,
+    zip_safe=False,
+    license="MPL2",
+)