diff --git a/bugbug/repository.py b/bugbug/repository.py index 2e5d5bf1..44332372 100644 --- a/bugbug/repository.py +++ b/bugbug/repository.py @@ -6,6 +6,7 @@ import argparse import concurrent.futures import copy +import io import itertools import json import logging @@ -21,7 +22,7 @@ import hglib from tqdm import tqdm from bugbug import db, utils -from bugbug.utils import LMDBDict +from bugbug.utils import LMDBDict, get_hgmo_patch logger = logging.getLogger(__name__) @@ -1011,6 +1012,48 @@ def clone(repo_dir): clean(repo_dir) +def apply_stack(repo_dir, stack, branch, default_base): + """Apply a stack of patches on a repository""" + assert len(stack) > 0, "Empty stack" + + # Start by updating the repository + clean(repo_dir) + + def has_revision(revision): + try: + hg.identify(revision) + return True + except hglib.error.CommandError: + return False + + with hglib.open(repo_dir) as hg: + + # Find the base revision to apply all the patches onto + # Use first parent from first patch if all its parents are available + # Otherwise fallback on tip + parents = stack[0]["parents"] + assert len(parents) > 0, "No parents found for first patch" + if all(map(has_revision, parents)): + base = parents[0] + else: + # Some repositories need to have the exact parent to apply + if default_base is None: + raise Exception("Parents are not available, cannot apply this stack") + + base = default_base + + # Update to base revision + logger.info(f"Will apply stack on {base}") + hg.update(base, clean=True) + + # Apply all the patches in the stack + for rev in stack: + node = rev["node"] + logger.info(f"Applying patch for {node}") + patch = get_hgmo_patch(branch, node) + hg.import_(patches=io.BytesIO(patch.encode("utf-8")), user="bugbug") + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("repository_dir", help="Path to the repository", action="store") diff --git a/bugbug/utils.py b/bugbug/utils.py index 08e48f44..2c77e7ca 100644 --- a/bugbug/utils.py +++ b/bugbug/utils.py @@ -370,3 +370,11 @@ class ThreadPoolExecutorResult(concurrent.futures.ThreadPoolExecutor): future.cancel() raise e return super(ThreadPoolExecutorResult, self).__exit__(*args) + + +def get_hgmo_patch(branch: str, revision: str) -> str: + """Load a patch for a given revision""" + url = f"https://hg.mozilla.org/{branch}/raw-rev/{revision}" + r = requests.get(url) + r.raise_for_status() + return r.text diff --git a/http_service/bugbug_http/__init__.py b/http_service/bugbug_http/__init__.py index dd4650c8..979f3fd6 100644 --- a/http_service/bugbug_http/__init__.py +++ b/http_service/bugbug_http/__init__.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- import os +import tempfile ALLOW_MISSING_MODELS = bool(int(os.environ.get("BUGBUG_ALLOW_MISSING_MODELS", "0"))) + +REPO_DIR = os.environ.get( + "BUGBUG_REPO_DIR", os.path.join(tempfile.gettempdir(), "bugbug-hg") +) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 818370d5..63536221 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -4,23 +4,18 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import logging -import os -import tempfile import bugbug_http.models from bugbug import db, repository, test_scheduling -from bugbug_http import ALLOW_MISSING_MODELS +from bugbug_http import ALLOW_MISSING_MODELS, REPO_DIR logger = logging.getLogger(__name__) def boot_worker(): # Clone mozilla central - repo_dir = os.environ.get( - "BUGBUG_REPO_DIR", os.path.join(tempfile.gettempdir(), "bugbug-hg") - ) - logger.info(f"Cloning mozilla-central in {repo_dir}...") - repository.clone(repo_dir) + logger.info(f"Cloning mozilla-central in {REPO_DIR}...") + repository.clone(REPO_DIR) # Download test scheduling DB support files. logger.info("Downloading test scheduling DB support files...") @@ -54,7 +49,7 @@ def boot_worker(): rev_start = "children({})".format(commit["node"]) logger.info("Updating commits DB...") - repository.download_commits(repo_dir, rev_start) + repository.download_commits(REPO_DIR, rev_start) # Preload models bugbug_http.models.preload_models() diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 1099e1ef..b48f09e1 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -16,7 +16,9 @@ from redis import Redis from bugbug import bugzilla from bugbug.model import Model from bugbug.models import load_model +from bugbug.repository import apply_stack from bugbug_http import ALLOW_MISSING_MODELS +from bugbug_http.utils import get_hgmo_stack logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger() @@ -30,11 +32,7 @@ MODELS_NAMES = [ "testlabelselect", "testgroupselect", ] -MODELS_TO_PRELOAD = [ - "component", - "testlabelselect", - "testgroupselect", -] +MODELS_TO_PRELOAD = ["component", "testlabelselect", "testgroupselect"] DEFAULT_EXPIRATION_TTL = 7 * 24 * 3600 # A week @@ -145,18 +143,28 @@ def classify_bug(model_name, bug_ids, bugzilla_token): def schedule_tests(branch, rev): from bugbug_http.app import JobInfo + from bugbug_http import REPO_DIR job = JobInfo(schedule_tests, branch, rev) - LOGGER.debug("Processing {job}") + LOGGER.debug(f"Processing {job}") - url = f"https://hg.mozilla.org/{branch}/json-automationrelevance/{rev}" - r = requests.get(url) - - if r.status_code == 404: - LOGGER.warning(f"Push not found at {url}!") + # Load the full stack of patches leading to that revision + try: + stack = get_hgmo_stack(branch, rev) + except requests.exceptions.RequestException: + LOGGER.warning(f"Push not found for {branch} @ {rev}!") return "NOK" - first_rev = r.json()["changesets"][0]["node"] + # Apply the stack on the local repository + # Autoland should always rebase on top of parents, never on tip + default_base = "tip" if branch != "integration/autoland" else None + try: + apply_stack(REPO_DIR, stack, branch, default_base) + except Exception as e: + LOGGER.warning(f"Failed to apply stack {branch} @ {rev}: {e}") + return "NOK" + + first_rev = stack[0]["node"] if first_rev != rev: revset = f"{first_rev}::{rev}" else: diff --git a/http_service/bugbug_http/utils.py b/http_service/bugbug_http/utils.py index 68332038..ca96d2d3 100644 --- a/http_service/bugbug_http/utils.py +++ b/http_service/bugbug_http/utils.py @@ -22,3 +22,11 @@ def get_bugzilla_http_client(): http_client.mount(bugzilla_url, HTTPAdapter(max_retries=retries)) return http_client, bugzilla_api_url + + +def get_hgmo_stack(branch: str, revision: str) -> list: + """Load descriptions of patches in the stack for a given revision""" + url = f"https://hg.mozilla.org/{branch}/json-automationrelevance/{revision}" + r = requests.get(url) + r.raise_for_status() + return r.json()["changesets"] diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index d4a52ca8..fe05d622 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -5,14 +5,33 @@ import json import logging +import os +import re from collections import defaultdict from datetime import datetime +import hglib import pytest +import responses from rq.exceptions import NoSuchJobError +import bugbug.repository +import bugbug_http +import bugbug_http.models from bugbug_http import app +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") + + +@pytest.fixture +def get_fixture_path(): + def _get_fixture_path(path): + path = os.path.join(FIXTURES_DIR, path) + assert os.path.exists(path), f"Missing fixture {path}" + return path + + return _get_fixture_path + @pytest.fixture def client(): @@ -41,6 +60,7 @@ def patch_resources(monkeypatch, jobs): def __init__(self): self.data = {} + self.expirations = {} def set(self, k, v): # keep track of job ids for testing purposes @@ -67,6 +87,9 @@ def patch_resources(monkeypatch, jobs): def ping(self): pass + def expire(self, key, expiration): + self.expirations[key] = expiration + class QueueMock: """Mock class to mimic rq.Queue.""" @@ -108,7 +131,9 @@ def patch_resources(monkeypatch, jobs): pass app.LOGGER.setLevel(logging.DEBUG) - monkeypatch.setattr(app, "redis_conn", RedisMock()) + _redis = RedisMock() + monkeypatch.setattr(app, "redis_conn", _redis) + monkeypatch.setattr(bugbug_http.models, "redis", _redis) monkeypatch.setattr(app, "q", QueueMock()) monkeypatch.setattr(app, "Job", JobMock) @@ -131,3 +156,78 @@ def add_change_time(): app.redis_conn.set(change_time_key, change_time) return inner + + +@pytest.fixture +def mock_hgmo(get_fixture_path, mock_repo): + """Mock HGMO API to get patches to apply""" + + def fake_raw_rev(request): + *repo, _, revision = request.path_url[1:].split("/") + repo = "-".join(repo) + + assert repo != "None", "Missing repo" + assert revision != "None", "Missing revision" + + mock_path = get_fixture_path(f"hgmo_{repo}/{revision}.diff") + with open(mock_path) as f: + content = f.read() + + return (200, {"Content-Type": "text/plain"}, content) + + def fake_json_relevance(request): + *repo, _, revision = request.path_url[1:].split("/") + repo = "-".join(repo) + + assert repo != "None", "Missing repo" + assert revision != "None", "Missing revision" + + mock_path = get_fixture_path(f"hgmo_{repo}/{revision}.json") + with open(mock_path) as f: + content = f.read() + + # Patch the hardcoded revisions + for log in mock_repo[1].log(): + log_id = log.rev.decode("utf-8") + node = log.node.decode("utf-8") + content = content.replace(f"BASE_HISTORY_{log_id}", node) + + return (200, {"Content-Type": "application/json"}, content) + + responses.add_callback( + responses.GET, + re.compile(r"^https?://(hgmo|hg\.mozilla\.org)/[\w\-\/]+/raw-rev/(\w+)"), + callback=fake_raw_rev, + ) + responses.add_callback( + responses.GET, + re.compile( + r"^https?://(hgmo|hg\.mozilla\.org)/[\w\-\/]+/json-automationrelevance/(\w+)" + ), + callback=fake_json_relevance, + ) + + +@pytest.fixture +def mock_repo(tmpdir, monkeypatch): + """Create an empty mercurial repo""" + repo_dir = tmpdir / "repo" + + # Setup the worker env to use that repo dir + monkeypatch.setattr(bugbug_http, "REPO_DIR", str(repo_dir)) + + # Silence the clean method + monkeypatch.setattr(bugbug.repository, "clean", lambda repo_dir: True) + + # Create the repo + hglib.init(str(repo_dir)) + + # Add several commits on a test file to create some history + test_file = repo_dir / "test.txt" + repo = hglib.open(str(repo_dir)) + for i in range(4): + test_file.write_text(f"Version {i}", encoding="utf-8") + repo.add([str(test_file).encode("utf-8")]) + repo.commit(f"Base history {i}", user="bugbug") + + return repo_dir, repo diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff b/http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff new file mode 100644 index 00000000..4cee19c0 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff @@ -0,0 +1,8 @@ +Parent 123 + +diff --git a/hello.txt b/hello.txt +new file mode 100644 +--- /dev/null ++++ b/hello.txt +@@ -0,0 +1,1 @@ ++Hello from autoland diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff b/http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff new file mode 100644 index 00000000..4378d143 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff @@ -0,0 +1,8 @@ +Target patch + +diff --git a/hello.txt b/hello.txt +--- a/hello.txt ++++ b/hello.txt +@@ -1,1 +1,2 @@ + Hello from autoland ++Nice to meet you diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/normal123.json b/http_service/tests/fixtures/hgmo_integration-autoland/normal123.json new file mode 100644 index 00000000..4f25f3c3 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_integration-autoland/normal123.json @@ -0,0 +1,18 @@ +{ + "changesets": [ + { + "node": "localParent123", + "parents": [ + "BASE_HISTORY_0" + ] + }, + { + "node": "normal123", + "parents": [ + "localParent123" + ] + } + ], + "visible": true +} + diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.json b/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.json new file mode 100644 index 00000000..c88ce51f --- /dev/null +++ b/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.json @@ -0,0 +1,11 @@ +{ + "changesets": [ + { + "node": "orphan123", + "parents": [ + "REVISION_DOES_NOT_EXIST" + ] + } + ], + "visible": true +} diff --git a/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff b/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff new file mode 100644 index 00000000..f138ace4 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff @@ -0,0 +1,10 @@ +Bug XYZ - Dummy patch + +diff --git a/test.txt b/test.txt +--- a/test.txt ++++ b/test.txt +@@ -1,1 +1,2 @@ +-Version 3 +\ No newline at end of file ++Version 3 ++This is a new line diff --git a/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.json b/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.json new file mode 100644 index 00000000..5b74e930 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.json @@ -0,0 +1,11 @@ +{ + "changesets": [ + { + "node": "12345deadbeef", + "parents": [ + "xxxxx" + ] + } + ], + "visible": true +} diff --git a/http_service/tests/fixtures/hgmo_try/bad123.diff b/http_service/tests/fixtures/hgmo_try/bad123.diff new file mode 100644 index 00000000..b19f76c9 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_try/bad123.diff @@ -0,0 +1,8 @@ +This patch will not apply cleanly + +diff --git a/hello.txt b/hello.txt +--- a/hello.txt ++++ b/hello.txt +@@ -1,1 +1,2 @@ + Does not exists ++Bad hunk diff --git a/http_service/tests/fixtures/hgmo_try/bad123.json b/http_service/tests/fixtures/hgmo_try/bad123.json new file mode 100644 index 00000000..d985e679 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_try/bad123.json @@ -0,0 +1,11 @@ +{ + "changesets": [ + { + "node": "bad123", + "parents": [ + "BASE_HISTORY_2" + ] + } + ], + "visible": true +} diff --git a/http_service/tests/fixtures/hgmo_try/localParent456.diff b/http_service/tests/fixtures/hgmo_try/localParent456.diff new file mode 100644 index 00000000..6a8d2b36 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_try/localParent456.diff @@ -0,0 +1,8 @@ +Parent 456 + +diff --git a/hello.txt b/hello.txt +new file mode 100644 +--- /dev/null ++++ b/hello.txt +@@ -0,0 +1,1 @@ ++Hello from try diff --git a/http_service/tests/fixtures/hgmo_try/normal456.diff b/http_service/tests/fixtures/hgmo_try/normal456.diff new file mode 100644 index 00000000..c80399ec --- /dev/null +++ b/http_service/tests/fixtures/hgmo_try/normal456.diff @@ -0,0 +1,8 @@ +Target patch + +diff --git a/hello.txt b/hello.txt +--- a/hello.txt ++++ b/hello.txt +@@ -1,1 +1,2 @@ + Hello from try ++Nice to meet you diff --git a/http_service/tests/fixtures/hgmo_try/normal456.json b/http_service/tests/fixtures/hgmo_try/normal456.json new file mode 100644 index 00000000..9a45082f --- /dev/null +++ b/http_service/tests/fixtures/hgmo_try/normal456.json @@ -0,0 +1,17 @@ +{ + "changesets": [ + { + "node": "localParent456", + "parents": [ + "BASE_HISTORY_1" + ] + }, + { + "node": "normal456", + "parents": [ + "localParent456" + ] + } + ], + "visible": true +} diff --git a/http_service/tests/fixtures/hgmo_try/orphan456.diff b/http_service/tests/fixtures/hgmo_try/orphan456.diff new file mode 100644 index 00000000..9c8ea0a9 --- /dev/null +++ b/http_service/tests/fixtures/hgmo_try/orphan456.diff @@ -0,0 +1,8 @@ +Orphan 456 + +diff --git a/hello.txt b/hello.txt +new file mode 100644 +--- /dev/null ++++ b/hello.txt +@@ -0,0 +1,1 @@ ++Hello from try - single orphan diff --git a/http_service/tests/fixtures/hgmo_try/orphan456.json b/http_service/tests/fixtures/hgmo_try/orphan456.json new file mode 100644 index 00000000..4774ab3c --- /dev/null +++ b/http_service/tests/fixtures/hgmo_try/orphan456.json @@ -0,0 +1,11 @@ +{ + "changesets": [ + { + "node": "orphan456", + "parents": [ + "REVISION_DOES_NOT_EXIST" + ] + } + ], + "visible": true +} diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py new file mode 100644 index 00000000..bc5bf6dd --- /dev/null +++ b/http_service/tests/test_schedule_tests.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +from bugbug_http.models import schedule_tests + + +def test_simple_schedule(patch_resources, mock_hgmo, mock_repo): + + # The repo should be almost empty at first + repo_dir, repo = mock_repo + assert len(repo.log()) == 4 + test_txt = repo_dir / "test.txt" + assert test_txt.exists() + assert test_txt.read_text("utf-8") == "Version 3" + + # Scheduling a test on a revision should apply changes in the repo + assert schedule_tests("mozilla-central", "12345deadbeef") == "OK" + + # Check changes have been applied + assert len(repo.log()) == 5 + assert test_txt.read_text("utf-8") == "Version 3\nThis is a new line\n" + + +@pytest.mark.parametrize( + "branch, revision, result, final_log", + [ + # patch from autoland based on local parent n°0 + ( + "integration/autoland", + "normal123", + "OK", + ["Target patch", "Parent 123", "Base history 0"], + ), + # patch from autoland where parent is not available + # so the patch is rejected as we need the parents on autoland + # and no changes is made on the repository + ( + "integration/autoland", + "orphan123", + "NOK", + ["Base history 3", "Base history 2", "Base history 1", "Base history 0"], + ), + # patch from try based on local parent n°1 + ( + "try", + "normal456", + "OK", + ["Target patch", "Parent 456", "Base history 1", "Base history 0"], + ), + # patch from try where parent is not available + # so the patch is applied on top of tip + ( + "try", + "orphan456", + "OK", + [ + "Orphan 456", + "Base history 3", + "Base history 2", + "Base history 1", + "Base history 0", + ], + ), + # bad patch that does not apply + # The repository is updated to the local base revision + # even if the patch does not apply afterward + ( + "try", + "bad123", + "NOK", + ["Base history 2", "Base history 1", "Base history 0"], + ), + ], +) +def test_schedule( + branch, revision, result, final_log, patch_resources, mock_hgmo, mock_repo +): + + # The repo should only have the base commits + repo_dir, repo = mock_repo + logs = repo.log(follow=True) + assert len(logs) == 4 + assert [l.desc.decode("utf-8") for l in logs] == [ + "Base history 3", + "Base history 2", + "Base history 1", + "Base history 0", + ] + + # Schedule tests for parametrized revision + assert schedule_tests(branch, revision) == result + + # Now check the log has evolved + assert final_log == [l.desc.decode("utf-8") for l in repo.log(follow=True)]