Apply mercurial patches on local repo when scheduling tests (#1369)

Fixes #1301

bugbug/repository.py

@@ -6,6 +6,7 @@
 import argparse
 import concurrent.futures
 import copy
+import io
 import itertools
 import json
 import logging
@@ -21,7 +22,7 @@ import hglib
 from tqdm import tqdm
 
 from bugbug import db, utils
-from bugbug.utils import LMDBDict
+from bugbug.utils import LMDBDict, get_hgmo_patch
 
 logger = logging.getLogger(__name__)
 
@@ -1011,6 +1012,48 @@ def clone(repo_dir):
     clean(repo_dir)
 
 
+def apply_stack(repo_dir, stack, branch, default_base):
+    """Apply a stack of patches on a repository"""
+    assert len(stack) > 0, "Empty stack"
+
+    # Start by updating the repository
+    clean(repo_dir)
+
+    def has_revision(revision):
+        try:
+            hg.identify(revision)
+            return True
+        except hglib.error.CommandError:
+            return False
+
+    with hglib.open(repo_dir) as hg:
+
+        # Find the base revision to apply all the patches onto
+        # Use first parent from first patch if all its parents are available
+        # Otherwise fallback on tip
+        parents = stack[0]["parents"]
+        assert len(parents) > 0, "No parents found for first patch"
+        if all(map(has_revision, parents)):
+            base = parents[0]
+        else:
+            # Some repositories need to have the exact parent to apply
+            if default_base is None:
+                raise Exception("Parents are not available, cannot apply this stack")
+
+            base = default_base
+
+        # Update to base revision
+        logger.info(f"Will apply stack on {base}")
+        hg.update(base, clean=True)
+
+        # Apply all the patches in the stack
+        for rev in stack:
+            node = rev["node"]
+            logger.info(f"Applying patch for {node}")
+            patch = get_hgmo_patch(branch, node)
+            hg.import_(patches=io.BytesIO(patch.encode("utf-8")), user="bugbug")
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("repository_dir", help="Path to the repository", action="store")

bugbug/utils.py

@@ -370,3 +370,11 @@ class ThreadPoolExecutorResult(concurrent.futures.ThreadPoolExecutor):
                 future.cancel()
             raise e
         return super(ThreadPoolExecutorResult, self).__exit__(*args)
+
+
+def get_hgmo_patch(branch: str, revision: str) -> str:
+    """Load a patch for a given revision"""
+    url = f"https://hg.mozilla.org/{branch}/raw-rev/{revision}"
+    r = requests.get(url)
+    r.raise_for_status()
+    return r.text

http_service/bugbug_http/__init__.py

@@ -1,4 +1,9 @@
 # -*- coding: utf-8 -*-
 import os
+import tempfile
 
 ALLOW_MISSING_MODELS = bool(int(os.environ.get("BUGBUG_ALLOW_MISSING_MODELS", "0")))
+
+REPO_DIR = os.environ.get(
+    "BUGBUG_REPO_DIR", os.path.join(tempfile.gettempdir(), "bugbug-hg")
+)

http_service/bugbug_http/boot.py

@@ -4,23 +4,18 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import logging
-import os
-import tempfile
 
 import bugbug_http.models
 from bugbug import db, repository, test_scheduling
-from bugbug_http import ALLOW_MISSING_MODELS
+from bugbug_http import ALLOW_MISSING_MODELS, REPO_DIR
 
 logger = logging.getLogger(__name__)
 
 
 def boot_worker():
     # Clone mozilla central
-    repo_dir = os.environ.get(
-        "BUGBUG_REPO_DIR", os.path.join(tempfile.gettempdir(), "bugbug-hg")
-    )
-    logger.info(f"Cloning mozilla-central in {repo_dir}...")
-    repository.clone(repo_dir)
+    logger.info(f"Cloning mozilla-central in {REPO_DIR}...")
+    repository.clone(REPO_DIR)
 
     # Download test scheduling DB support files.
     logger.info("Downloading test scheduling DB support files...")
@@ -54,7 +49,7 @@ def boot_worker():
 
         rev_start = "children({})".format(commit["node"])
         logger.info("Updating commits DB...")
-        repository.download_commits(repo_dir, rev_start)
+        repository.download_commits(REPO_DIR, rev_start)
 
     # Preload models
     bugbug_http.models.preload_models()

http_service/bugbug_http/models.py

@@ -16,7 +16,9 @@ from redis import Redis
 from bugbug import bugzilla
 from bugbug.model import Model
 from bugbug.models import load_model
+from bugbug.repository import apply_stack
 from bugbug_http import ALLOW_MISSING_MODELS
+from bugbug_http.utils import get_hgmo_stack
 
 logging.basicConfig(level=logging.INFO)
 LOGGER = logging.getLogger()
@@ -30,11 +32,7 @@ MODELS_NAMES = [
     "testlabelselect",
     "testgroupselect",
 ]
-MODELS_TO_PRELOAD = [
-    "component",
-    "testlabelselect",
-    "testgroupselect",
-]
+MODELS_TO_PRELOAD = ["component", "testlabelselect", "testgroupselect"]
 DEFAULT_EXPIRATION_TTL = 7 * 24 * 3600  # A week
 
 
@@ -145,18 +143,28 @@ def classify_bug(model_name, bug_ids, bugzilla_token):
 
 def schedule_tests(branch, rev):
     from bugbug_http.app import JobInfo
+    from bugbug_http import REPO_DIR
 
     job = JobInfo(schedule_tests, branch, rev)
-    LOGGER.debug("Processing {job}")
+    LOGGER.debug(f"Processing {job}")
 
-    url = f"https://hg.mozilla.org/{branch}/json-automationrelevance/{rev}"
-    r = requests.get(url)
-
-    if r.status_code == 404:
-        LOGGER.warning(f"Push not found at {url}!")
+    # Load the full stack of patches leading to that revision
+    try:
+        stack = get_hgmo_stack(branch, rev)
+    except requests.exceptions.RequestException:
+        LOGGER.warning(f"Push not found for {branch} @ {rev}!")
         return "NOK"
 
-    first_rev = r.json()["changesets"][0]["node"]
+    # Apply the stack on the local repository
+    # Autoland should always rebase on top of parents, never on tip
+    default_base = "tip" if branch != "integration/autoland" else None
+    try:
+        apply_stack(REPO_DIR, stack, branch, default_base)
+    except Exception as e:
+        LOGGER.warning(f"Failed to apply stack {branch} @ {rev}: {e}")
+        return "NOK"
+
+    first_rev = stack[0]["node"]
     if first_rev != rev:
         revset = f"{first_rev}::{rev}"
     else:

http_service/bugbug_http/utils.py

@@ -22,3 +22,11 @@ def get_bugzilla_http_client():
     http_client.mount(bugzilla_url, HTTPAdapter(max_retries=retries))
 
     return http_client, bugzilla_api_url
+
+
+def get_hgmo_stack(branch: str, revision: str) -> list:
+    """Load descriptions of patches in the stack for a given revision"""
+    url = f"https://hg.mozilla.org/{branch}/json-automationrelevance/{revision}"
+    r = requests.get(url)
+    r.raise_for_status()
+    return r.json()["changesets"]

http_service/tests/conftest.py

@@ -5,14 +5,33 @@
 
 import json
 import logging
+import os
+import re
 from collections import defaultdict
 from datetime import datetime
 
+import hglib
 import pytest
+import responses
 from rq.exceptions import NoSuchJobError
 
+import bugbug.repository
+import bugbug_http
+import bugbug_http.models
 from bugbug_http import app
 
+FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures")
+
+
+@pytest.fixture
+def get_fixture_path():
+    def _get_fixture_path(path):
+        path = os.path.join(FIXTURES_DIR, path)
+        assert os.path.exists(path), f"Missing fixture {path}"
+        return path
+
+    return _get_fixture_path
+
 
 @pytest.fixture
 def client():
@@ -41,6 +60,7 @@ def patch_resources(monkeypatch, jobs):
 
         def __init__(self):
             self.data = {}
+            self.expirations = {}
 
         def set(self, k, v):
             # keep track of job ids for testing purposes
@@ -67,6 +87,9 @@ def patch_resources(monkeypatch, jobs):
         def ping(self):
             pass
 
+        def expire(self, key, expiration):
+            self.expirations[key] = expiration
+
     class QueueMock:
         """Mock class to mimic rq.Queue."""
 
@@ -108,7 +131,9 @@ def patch_resources(monkeypatch, jobs):
             pass
 
     app.LOGGER.setLevel(logging.DEBUG)
-    monkeypatch.setattr(app, "redis_conn", RedisMock())
+    _redis = RedisMock()
+    monkeypatch.setattr(app, "redis_conn", _redis)
+    monkeypatch.setattr(bugbug_http.models, "redis", _redis)
     monkeypatch.setattr(app, "q", QueueMock())
     monkeypatch.setattr(app, "Job", JobMock)
 
@@ -131,3 +156,78 @@ def add_change_time():
         app.redis_conn.set(change_time_key, change_time)
 
     return inner
+
+
+@pytest.fixture
+def mock_hgmo(get_fixture_path, mock_repo):
+    """Mock HGMO API to get patches to apply"""
+
+    def fake_raw_rev(request):
+        *repo, _, revision = request.path_url[1:].split("/")
+        repo = "-".join(repo)
+
+        assert repo != "None", "Missing repo"
+        assert revision != "None", "Missing revision"
+
+        mock_path = get_fixture_path(f"hgmo_{repo}/{revision}.diff")
+        with open(mock_path) as f:
+            content = f.read()
+
+        return (200, {"Content-Type": "text/plain"}, content)
+
+    def fake_json_relevance(request):
+        *repo, _, revision = request.path_url[1:].split("/")
+        repo = "-".join(repo)
+
+        assert repo != "None", "Missing repo"
+        assert revision != "None", "Missing revision"
+
+        mock_path = get_fixture_path(f"hgmo_{repo}/{revision}.json")
+        with open(mock_path) as f:
+            content = f.read()
+
+        # Patch the hardcoded revisions
+        for log in mock_repo[1].log():
+            log_id = log.rev.decode("utf-8")
+            node = log.node.decode("utf-8")
+            content = content.replace(f"BASE_HISTORY_{log_id}", node)
+
+        return (200, {"Content-Type": "application/json"}, content)
+
+    responses.add_callback(
+        responses.GET,
+        re.compile(r"^https?://(hgmo|hg\.mozilla\.org)/[\w\-\/]+/raw-rev/(\w+)"),
+        callback=fake_raw_rev,
+    )
+    responses.add_callback(
+        responses.GET,
+        re.compile(
+            r"^https?://(hgmo|hg\.mozilla\.org)/[\w\-\/]+/json-automationrelevance/(\w+)"
+        ),
+        callback=fake_json_relevance,
+    )
+
+
+@pytest.fixture
+def mock_repo(tmpdir, monkeypatch):
+    """Create an empty mercurial repo"""
+    repo_dir = tmpdir / "repo"
+
+    # Setup the worker env to use that repo dir
+    monkeypatch.setattr(bugbug_http, "REPO_DIR", str(repo_dir))
+
+    # Silence the clean method
+    monkeypatch.setattr(bugbug.repository, "clean", lambda repo_dir: True)
+
+    # Create the repo
+    hglib.init(str(repo_dir))
+
+    # Add several commits on a test file to create some history
+    test_file = repo_dir / "test.txt"
+    repo = hglib.open(str(repo_dir))
+    for i in range(4):
+        test_file.write_text(f"Version {i}", encoding="utf-8")
+        repo.add([str(test_file).encode("utf-8")])
+        repo.commit(f"Base history {i}", user="bugbug")
+
+    return repo_dir, repo

http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff

@@ -0,0 +1,8 @@
+Parent 123
+
+diff --git a/hello.txt b/hello.txt
+new file mode 100644
+--- /dev/null
++++ b/hello.txt
+@@ -0,0 +1,1 @@
++Hello from autoland

http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff

@@ -0,0 +1,8 @@
+Target patch
+
+diff --git a/hello.txt b/hello.txt
+--- a/hello.txt
++++ b/hello.txt
+@@ -1,1 +1,2 @@
+ Hello from autoland
++Nice to meet you

http_service/tests/fixtures/hgmo_integration-autoland/normal123.json

@@ -0,0 +1,18 @@
+{
+    "changesets": [
+        {
+            "node": "localParent123",
+            "parents": [
+                "BASE_HISTORY_0"
+            ]
+        },
+        {
+            "node": "normal123",
+            "parents": [
+                "localParent123"
+            ]
+        }
+    ],
+    "visible": true
+}
+

http_service/tests/fixtures/hgmo_integration-autoland/orphan123.json

@@ -0,0 +1,11 @@
+{
+    "changesets": [
+        {
+            "node": "orphan123",
+            "parents": [
+                "REVISION_DOES_NOT_EXIST"
+            ]
+        }
+    ],
+    "visible": true
+}

http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff

@@ -0,0 +1,10 @@
+Bug XYZ - Dummy patch
+
+diff --git a/test.txt b/test.txt
+--- a/test.txt
++++ b/test.txt
+@@ -1,1 +1,2 @@
+-Version 3
+\ No newline at end of file
++Version 3
++This is a new line

http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.json

@@ -0,0 +1,11 @@
+{
+    "changesets": [
+        {
+            "node": "12345deadbeef",
+            "parents": [
+                "xxxxx"
+            ]
+        }
+    ],
+    "visible": true
+}

http_service/tests/fixtures/hgmo_try/bad123.diff

@@ -0,0 +1,8 @@
+This patch will not apply cleanly
+
+diff --git a/hello.txt b/hello.txt
+--- a/hello.txt
++++ b/hello.txt
+@@ -1,1 +1,2 @@
+ Does not exists
++Bad hunk

http_service/tests/fixtures/hgmo_try/bad123.json

@@ -0,0 +1,11 @@
+{
+    "changesets": [
+        {
+            "node": "bad123",
+            "parents": [
+                "BASE_HISTORY_2"
+            ]
+        }
+    ],
+    "visible": true
+}

http_service/tests/fixtures/hgmo_try/localParent456.diff

@@ -0,0 +1,8 @@
+Parent 456
+
+diff --git a/hello.txt b/hello.txt
+new file mode 100644
+--- /dev/null
++++ b/hello.txt
+@@ -0,0 +1,1 @@
++Hello from try

http_service/tests/fixtures/hgmo_try/normal456.diff

@@ -0,0 +1,8 @@
+Target patch
+
+diff --git a/hello.txt b/hello.txt
+--- a/hello.txt
++++ b/hello.txt
+@@ -1,1 +1,2 @@
+ Hello from try
++Nice to meet you

http_service/tests/fixtures/hgmo_try/normal456.json

@@ -0,0 +1,17 @@
+{
+    "changesets": [
+        {
+            "node": "localParent456",
+            "parents": [
+                "BASE_HISTORY_1"
+            ]
+        },
+        {
+            "node": "normal456",
+            "parents": [
+                "localParent456"
+            ]
+        }
+    ],
+    "visible": true
+}

http_service/tests/fixtures/hgmo_try/orphan456.diff

@@ -0,0 +1,8 @@
+Orphan 456
+
+diff --git a/hello.txt b/hello.txt
+new file mode 100644
+--- /dev/null
++++ b/hello.txt
+@@ -0,0 +1,1 @@
++Hello from try - single orphan

http_service/tests/fixtures/hgmo_try/orphan456.json

@@ -0,0 +1,11 @@
+{
+    "changesets": [
+        {
+            "node": "orphan456",
+            "parents": [
+                "REVISION_DOES_NOT_EXIST"
+            ]
+        }
+    ],
+    "visible": true
+}

http_service/tests/test_schedule_tests.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pytest
+
+from bugbug_http.models import schedule_tests
+
+
+def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
+
+    # The repo should be almost empty at first
+    repo_dir, repo = mock_repo
+    assert len(repo.log()) == 4
+    test_txt = repo_dir / "test.txt"
+    assert test_txt.exists()
+    assert test_txt.read_text("utf-8") == "Version 3"
+
+    # Scheduling a test on a revision should apply changes in the repo
+    assert schedule_tests("mozilla-central", "12345deadbeef") == "OK"
+
+    # Check changes have been applied
+    assert len(repo.log()) == 5
+    assert test_txt.read_text("utf-8") == "Version 3\nThis is a new line\n"
+
+
+@pytest.mark.parametrize(
+    "branch, revision, result, final_log",
+    [
+        # patch from autoland based on local parent n°0
+        (
+            "integration/autoland",
+            "normal123",
+            "OK",
+            ["Target patch", "Parent 123", "Base history 0"],
+        ),
+        # patch from autoland where parent is not available
+        # so the patch is rejected as we need the parents on autoland
+        # and no changes is made on the repository
+        (
+            "integration/autoland",
+            "orphan123",
+            "NOK",
+            ["Base history 3", "Base history 2", "Base history 1", "Base history 0"],
+        ),
+        # patch from try based on local parent n°1
+        (
+            "try",
+            "normal456",
+            "OK",
+            ["Target patch", "Parent 456", "Base history 1", "Base history 0"],
+        ),
+        # patch from try where parent is not available
+        # so the patch is applied on top of tip
+        (
+            "try",
+            "orphan456",
+            "OK",
+            [
+                "Orphan 456",
+                "Base history 3",
+                "Base history 2",
+                "Base history 1",
+                "Base history 0",
+            ],
+        ),
+        # bad patch that does not apply
+        # The repository is updated to the local base revision
+        # even if the patch does not apply afterward
+        (
+            "try",
+            "bad123",
+            "NOK",
+            ["Base history 2", "Base history 1", "Base history 0"],
+        ),
+    ],
+)
+def test_schedule(
+    branch, revision, result, final_log, patch_resources, mock_hgmo, mock_repo
+):
+
+    # The repo should only have the base commits
+    repo_dir, repo = mock_repo
+    logs = repo.log(follow=True)
+    assert len(logs) == 4
+    assert [l.desc.decode("utf-8") for l in logs] == [
+        "Base history 3",
+        "Base history 2",
+        "Base history 1",
+        "Base history 0",
+    ]
+
+    # Schedule tests for parametrized revision
+    assert schedule_tests(branch, revision) == result
+
+    # Now check the log has evolved
+    assert final_log == [l.desc.decode("utf-8") for l in repo.log(follow=True)]