From f034fb6a56de1782600aecc4b11f037a6f433db7 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 25 Sep 2019 11:45:44 +0200 Subject: [PATCH] Papertrail support (#115) --- .isort.cfg | 2 +- .taskcluster.yml | 17 ++++---- bot/Dockerfile | 6 --- bot/code_review_bot/__init__.py | 2 +- bot/code_review_bot/cli.py | 5 ++- bot/code_review_bot/revisions.py | 2 +- bot/code_review_bot/workflow.py | 2 +- bot/docker/Dockerfile | 8 ++++ bot/docker/bootstrap.sh | 12 ++++++ bot/requirements.txt | 4 +- bot/tests/test_tools.py | 2 +- events/code_review_events/cli.py | 5 ++- events/docker/Dockerfile | 8 ++-- events/requirements.txt | 1 + events/setup.py | 20 ++++++++- .../code_review_tools}/__init__.py | 0 .../tools => tools/code_review_tools}/log.py | 2 +- .../code_review_tools}/taskcluster.py | 0 tools/requirements.txt | 4 ++ tools/setup.py | 41 +++++++++++++++++++ 20 files changed, 112 insertions(+), 31 deletions(-) delete mode 100644 bot/Dockerfile create mode 100644 bot/docker/Dockerfile create mode 100755 bot/docker/bootstrap.sh rename {bot/code_review_bot/tools => tools/code_review_tools}/__init__.py (100%) rename {bot/code_review_bot/tools => tools/code_review_tools}/log.py (97%) rename {bot/code_review_bot/tools => tools/code_review_tools}/taskcluster.py (100%) create mode 100644 tools/requirements.txt create mode 100644 tools/setup.py diff --git a/.isort.cfg b/.isort.cfg index d450176b..af6f66c4 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] -known_first_party = code_review_bot,code_review_events,conftest +known_first_party = code_review_bot,code_review_tools,code_review_events,conftest known_third_party = influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,requests,responses,setuptools,structlog,taskcluster,toml force_single_line = True default_section=FIRSTPARTY diff --git a/.taskcluster.yml b/.taskcluster.yml index 0a12f0c2..edcbbdb7 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -45,8 +45,9 @@ tasks: command: - sh - -lxce - - "git clone --quiet ${repository} /src && cd /src/bot && git checkout ${head_rev} -b checks && - pip install . && pip install -r requirements-dev.txt && + - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && + cd /src/tools && pip install . && + cd /src/bot && pip install . && pip install -r requirements-dev.txt && cd /src && pre-commit run -a" metadata: name: "Code Review Bot checks: linting" @@ -66,8 +67,9 @@ tasks: - sh - -lxce - "apk add git --quiet && - git clone --quiet ${repository} /src && cd /src/bot && git checkout ${head_rev} -b checks && - pip install . && pip install -r requirements-dev.txt && + git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && + cd /src/tools && pip install . && + cd /src/bot && pip install . && pip install -r requirements-dev.txt && pytest -v" metadata: name: "Code Review Bot checks: unit tests" @@ -86,8 +88,9 @@ tasks: command: - sh - -lxce - - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && cd /src/events && - pip install -r requirements.txt && pip install . && pip install -r requirements-dev.txt && + - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && + cd /src/tools && pip install . && + cd /src/events && pip install . && pip install -r requirements-dev.txt && pytest -v" metadata: name: "Code Review Events checks: unit tests" @@ -147,7 +150,7 @@ tasks: - "${head_rev}" - --write - /bot.tar - - bot/Dockerfile + - bot/docker/Dockerfile artifacts: public/code-review-bot.tar: expires: {$fromNow: '2 weeks'} diff --git a/bot/Dockerfile b/bot/Dockerfile deleted file mode 100644 index df3914c7..00000000 --- a/bot/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM python:3-alpine - -COPY bot /src -RUN pip install /src - -CMD ["code-review-bot"] diff --git a/bot/code_review_bot/__init__.py b/bot/code_review_bot/__init__.py index 40c54b1c..a74e9518 100644 --- a/bot/code_review_bot/__init__.py +++ b/bot/code_review_bot/__init__.py @@ -9,7 +9,7 @@ import enum from code_review_bot.config import Publication from code_review_bot.config import settings from code_review_bot.stats import InfluxDb -from code_review_bot.tools.taskcluster import TaskclusterConfig +from code_review_tools.taskcluster import TaskclusterConfig CLANG_TIDY = "clang-tidy" CLANG_FORMAT = "clang-format" diff --git a/bot/code_review_bot/cli.py b/bot/code_review_bot/cli.py index 2d9df256..0f73c202 100644 --- a/bot/code_review_bot/cli.py +++ b/bot/code_review_bot/cli.py @@ -17,8 +17,8 @@ from code_review_bot import taskcluster from code_review_bot.config import settings from code_review_bot.report import get_reporters from code_review_bot.revisions import Revision -from code_review_bot.tools.log import init_logger from code_review_bot.workflow import Workflow +from code_review_tools.log import init_logger logger = structlog.get_logger(__name__) @@ -57,7 +57,8 @@ def main(): ) init_logger( - config.PROJECT_NAME, + "bot", + channel=taskcluster.secrets.get("APP_CHANNEL", "dev"), PAPERTRAIL_HOST=taskcluster.secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=taskcluster.secrets.get("PAPERTRAIL_PORT"), SENTRY_DSN=taskcluster.secrets.get("SENTRY_DSN"), diff --git a/bot/code_review_bot/revisions.py b/bot/code_review_bot/revisions.py index 601289c8..864030ed 100644 --- a/bot/code_review_bot/revisions.py +++ b/bot/code_review_bot/revisions.py @@ -16,7 +16,7 @@ from code_review_bot import stats from code_review_bot.config import REPO_GECKO_TRY from code_review_bot.config import REPO_NSS_TRY from code_review_bot.config import settings -from code_review_bot.tools.taskcluster import create_blob_artifact +from code_review_tools.taskcluster import create_blob_artifact logger = structlog.get_logger(__name__) diff --git a/bot/code_review_bot/workflow.py b/bot/code_review_bot/workflow.py index 79ebab14..39abbc64 100644 --- a/bot/code_review_bot/workflow.py +++ b/bot/code_review_bot/workflow.py @@ -21,7 +21,7 @@ from code_review_bot.tasks.coverage import ZeroCoverageTask from code_review_bot.tasks.coverity import CoverityTask from code_review_bot.tasks.infer import InferTask from code_review_bot.tasks.lint import MozLintTask -from code_review_bot.tools.taskcluster import TASKCLUSTER_DATE_FORMAT +from code_review_tools.taskcluster import TASKCLUSTER_DATE_FORMAT logger = structlog.get_logger(__name__) diff --git a/bot/docker/Dockerfile b/bot/docker/Dockerfile new file mode 100644 index 00000000..49effc98 --- /dev/null +++ b/bot/docker/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3-alpine + +ADD tools /src/tools +ADD bot /src/bot + +RUN /src/bot/docker/bootstrap.sh + +CMD ["code-review-bot"] diff --git a/bot/docker/bootstrap.sh b/bot/docker/bootstrap.sh new file mode 100755 index 00000000..2683d616 --- /dev/null +++ b/bot/docker/bootstrap.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# Add build deps +apk --update add --virtual build-dependencies gcc musl-dev libffi-dev openssl-dev + +# Install python dependencies +cd /src/tools && python setup.py install +cd /src/bot && python setup.py install + +# Cleanup +apk del build-dependencies +rm -rf /var/cache/apk/* diff --git a/bot/requirements.txt b/bot/requirements.txt index 1e393921..69b2a494 100644 --- a/bot/requirements.txt +++ b/bot/requirements.txt @@ -1,7 +1,5 @@ +-e ../tools #egg=code-review-tools influxdb==5.2.3 libmozdata==0.1.59 -Logbook==1.5.2 parsepatch==0.1.3 -structlog==19.1.0 -taskcluster==16.2.0 toml==0.10.0 diff --git a/bot/tests/test_tools.py b/bot/tests/test_tools.py index 8612d9ce..8aee987d 100644 --- a/bot/tests/test_tools.py +++ b/bot/tests/test_tools.py @@ -5,7 +5,7 @@ import pytest -from code_review_bot.tools.taskcluster import TaskclusterConfig +from code_review_tools.taskcluster import TaskclusterConfig def test_taskcluster_service(): diff --git a/events/code_review_events/cli.py b/events/code_review_events/cli.py index fea31319..90a5568b 100644 --- a/events/code_review_events/cli.py +++ b/events/code_review_events/cli.py @@ -5,9 +5,9 @@ import tempfile import structlog from libmozevent import taskcluster_config -from libmozevent.log import init_logger from code_review_events.workflow import Events +from code_review_tools.log import init_logger logger = structlog.get_logger(__name__) @@ -45,7 +45,8 @@ def main(): ) init_logger( - "code_review_events", + "events", + channel=taskcluster_config.secrets.get("APP_CHANNEL", "dev"), PAPERTRAIL_HOST=taskcluster_config.secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=taskcluster_config.secrets.get("PAPERTRAIL_PORT"), SENTRY_DSN=taskcluster_config.secrets.get("SENTRY_DSN"), diff --git a/events/docker/Dockerfile b/events/docker/Dockerfile index 5920c9bf..450b31fd 100644 --- a/events/docker/Dockerfile +++ b/events/docker/Dockerfile @@ -1,9 +1,11 @@ FROM python:3-slim -ADD events /src +ADD tools /src/tools +ADD events /src/events -RUN cd /src && pip install --disable-pip-version-check --no-cache-dir -r requirements.txt && python setup.py install +RUN cd /src/tools && python setup.py install +RUN cd /src/events && python setup.py install -RUN /src/docker/bootstrap.sh +RUN /src/events/docker/bootstrap.sh CMD ["code-review-events"] diff --git a/events/requirements.txt b/events/requirements.txt index 4c95774e..95b07b6c 100644 --- a/events/requirements.txt +++ b/events/requirements.txt @@ -1 +1,2 @@ +-e ../tools #egg=code-review-tools libmozevent==1.0.0 diff --git a/events/setup.py b/events/setup.py index 30fd8f4a..062960fa 100644 --- a/events/setup.py +++ b/events/setup.py @@ -11,8 +11,24 @@ here = os.path.dirname(__file__) def read_requirements(file_): - with open(os.path.join(here, file_)) as f: - return sorted(list(set(line.split("#")[0].strip() for line in f))) + lines = [] + with open(file_) as f: + for line in f.readlines(): + line = line.strip() + if ( + line.startswith("-e ") + or line.startswith("http://") + or line.startswith("https://") + ): + extras = "" + if "[" in line: + extras = "[" + line.split("[")[1].split("]")[0] + "]" + line = line.split("#")[1].split("egg=")[1] + extras + elif line == "" or line.startswith("#") or line.startswith("-"): + continue + line = line.split("#")[0].strip() + lines.append(line) + return sorted(list(set(lines))) with open(os.path.join(here, "VERSION")) as f: diff --git a/bot/code_review_bot/tools/__init__.py b/tools/code_review_tools/__init__.py similarity index 100% rename from bot/code_review_bot/tools/__init__.py rename to tools/code_review_tools/__init__.py diff --git a/bot/code_review_bot/tools/log.py b/tools/code_review_tools/log.py similarity index 97% rename from bot/code_review_bot/tools/log.py rename to tools/code_review_tools/log.py index 8195b88f..ca3b4cde 100644 --- a/bot/code_review_bot/tools/log.py +++ b/tools/code_review_tools/log.py @@ -33,7 +33,7 @@ def setup_papertrail(project_name, channel, PAPERTRAIL_HOST, PAPERTRAIL_PORT): # Setup papertrail papertrail = logbook.SyslogHandler( - application_name=f"mozilla/release-services/{channel}/{project_name}", + application_name=f"code-review/{channel}/{project_name}", address=(PAPERTRAIL_HOST, int(PAPERTRAIL_PORT)), level=logbook.INFO, format_string="{record.time} {record.channel}: {record.message}", diff --git a/bot/code_review_bot/tools/taskcluster.py b/tools/code_review_tools/taskcluster.py similarity index 100% rename from bot/code_review_bot/tools/taskcluster.py rename to tools/code_review_tools/taskcluster.py diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000..68f5f210 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,4 @@ +Logbook==1.5.2 +structlog==19.1.0 +taskcluster==16.2.0 +raven==6.10.0 diff --git a/tools/setup.py b/tools/setup.py new file mode 100644 index 00000000..4edf24a1 --- /dev/null +++ b/tools/setup.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import setuptools + + +def read_requirements(file_): + lines = [] + with open(file_) as f: + for line in f.readlines(): + line = line.strip() + if ( + line.startswith("-e ") + or line.startswith("http://") + or line.startswith("https://") + ): + extras = "" + if "[" in line: + extras = "[" + line.split("[")[1].split("]")[0] + "]" + line = line.split("#")[1].split("egg=")[1] + extras + elif line == "" or line.startswith("#") or line.startswith("-"): + continue + line = line.split("#")[0].strip() + lines.append(line) + return sorted(list(set(lines))) + + +setuptools.setup( + name="code-review-tools", + version="0.1.0", + description="Support tools for Mozilla code review", + author="Mozilla Release Management", + author_email="release-mgmt-analysis@mozilla.com", + install_requires=read_requirements("requirements.txt"), + packages=setuptools.find_packages(), + include_package_data=True, + zip_safe=False, + license="MPL2", +)