Create a standalone logic to determine version status flags (#2156)

bugbot/bug/analyzer.py

@@ -0,0 +1,234 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from functools import cached_property
+from typing import Any, Iterable, NamedTuple
+
+from libmozdata import versions as lmdversions
+from libmozdata.bugzilla import Bugzilla
+
+from bugbot import utils
+
+
+class VersionStatus(NamedTuple):
+    """A representation of a version status flag"""
+
+    channel: str
+    version: int
+    status: str
+
+    @property
+    def flag(self) -> str:
+        return utils.get_flag(self.version, "status", self.channel)
+
+
+class BugAnalyzer:
+    """A class to analyze a bug"""
+
+    def __init__(self, bug: dict, store: "BugsStore"):
+        """Constructor
+
+        Args:
+            bug: The bug to analyze
+            store: The store of bugs
+        """
+        self._bug = bug
+        self._store = store
+
+    @property
+    def regressed_by_bugs(self) -> list["BugAnalyzer"]:
+        """The bugs that regressed the bug."""
+        return [
+            self._store.get_bug_by_id(bug_id) for bug_id in self._bug["regressed_by"]
+        ]
+
+    @property
+    def oldest_fixed_firefox_version(self) -> int | None:
+        """The oldest version of Firefox that was fixed by this bug."""
+        fixed_versions = sorted(
+            int(key[len("cf_status_firefox") :])
+            for key, value in self._bug.items()
+            if key.startswith("cf_status_firefox")
+            and "esr" not in key
+            and value in ("fixed", "verified")
+        )
+
+        if not fixed_versions:
+            return None
+
+        return fixed_versions[0]
+
+    @property
+    def latest_firefox_version_status(self) -> str | None:
+        """The version status for the latest version of Firefox.
+
+        The latest version is the highest version number that has a status flag
+        set (not `---`).
+        """
+        versions_status = sorted(
+            (int(key[len("cf_status_firefox") :]), value)
+            for key, value in self._bug.items()
+            if value != "---"
+            and key.startswith("cf_status_firefox")
+            and "esr" not in key
+        )
+
+        if not versions_status:
+            return None
+
+        return versions_status[-1][1]
+
+    def get_field(self, field: str) -> Any:
+        """Get a field value from the bug.
+
+        Args:
+            field: The field name.
+
+        Returns:
+            The field value. If the field is not found, `None` is returned.
+        """
+        return self._bug.get(field)
+
+    def detect_version_status_updates(self) -> list[VersionStatus] | None:
+        """Detect the status for the version flags that should be updated.
+
+        The status of the version flags is determined by the status of the
+        regressor bug.
+
+        Returns:
+            A list of `VersionStatus` objects.
+        """
+        if len(self._bug["regressed_by"]) > 1:
+            # Currently only bugs with one regressor are supported
+            return None
+
+        regressor_bug = self.regressed_by_bugs[0]
+        regressed_version = regressor_bug.oldest_fixed_firefox_version
+        if not regressed_version:
+            return None
+
+        fixed_version = self.oldest_fixed_firefox_version
+
+        # If the latest status flag is wontfix or fix-optional, we ignore
+        # setting flags with the status "affected" to newer versions.
+        is_latest_wontfix = self.latest_firefox_version_status in (
+            "wontfix",
+            "fix-optional",
+        )
+
+        flag_updates = []
+        for flag, channel, version in self._store.current_version_flags:
+            if flag not in self._bug and channel == "esr":
+                # It is okay if an ESR flag is absent (we try two, the current
+                # and the previous). However, the absence of other flags is a
+                # sign of something wrong.
+                continue
+            if self._bug[flag] != "---":
+                # We don't override existing flags
+                # XXX maybe check for consistency?
+                continue
+            if fixed_version and fixed_version <= version:
+                # Bug was fixed in an earlier version, don't set the flag
+                continue
+            if (
+                version >= regressed_version
+                # ESR: If the regressor was uplifted, so the regression affects
+                # this version.
+                or regressor_bug.get_field(flag) in ("fixed", "verified")
+            ):
+                if is_latest_wontfix:
+                    continue
+
+                flag_updates.append(VersionStatus(channel, version, "affected"))
+            else:
+                flag_updates.append(VersionStatus(channel, version, "unaffected"))
+
+        return flag_updates
+
+
+class BugNotInStoreError(LookupError):
+    """The bug was not found the bugs store."""
+
+
+class BugsStore:
+    """A class to retrieve bugs."""
+
+    def __init__(self, bugs: Iterable[dict], versions_map: dict[str, int] = None):
+        self.bugs = {bug["id"]: BugAnalyzer(bug, self) for bug in bugs}
+        self.versions_map = versions_map
+
+    def get_bug_by_id(self, bug_id: int) -> BugAnalyzer:
+        """Get a bug by its id.
+
+        Args:
+            bug_id: The id of the bug to retrieve.
+
+        Returns:
+            A `BugAnalyzer` object representing the bug.
+
+        Raises:
+            BugNotFoundError: The bug was not found in the store.
+        """
+        try:
+            return self.bugs[bug_id]
+        except KeyError as error:
+            raise BugNotInStoreError(f"Bug {bug_id} is not the bugs store") from error
+
+    def fetch_regressors(self, include_fields: list[str] = None):
+        """Fetches the regressors for all the bugs in the store.
+
+        Args:
+            include_fields: The fields to include when fetching the bugs.
+        """
+        bug_ids = {
+            bug_id
+            for bug in self.bugs.values()
+            if bug.get_field("regressed_by")
+            for bug_id in bug.get_field("regressed_by")
+            if bug_id not in self.bugs
+        }
+
+        if not bug_ids:
+            return
+
+        self.fetch_bugs(bug_ids, include_fields)
+
+    def fetch_bugs(self, bug_ids: Iterable[int], include_fields: list[str] = None):
+        """Fetches the bugs from Bugzilla.
+
+        Args:
+            bug_ids: The ids of the bugs to fetch.
+            include_fields: The fields to include when fetching the bugs.
+        """
+
+        def bug_handler(bugs):
+            for bug in bugs:
+                self.bugs[bug["id"]] = BugAnalyzer(bug, self)
+
+        Bugzilla(bug_ids, bughandler=bug_handler, include_fields=include_fields).wait()
+
+    @cached_property
+    def current_version_flags(self) -> list[tuple[str, str, int]]:
+        """The current version flags."""
+        active_versions = []
+
+        channel_version_map = (
+            self.versions_map if self.versions_map else lmdversions.get(base=True)
+        )
+        for channel in ("release", "beta", "nightly"):
+            version = int(channel_version_map[channel])
+            flag = utils.get_flag(version, "status", channel)
+            active_versions.append((flag, channel, version))
+
+        esr_versions = {
+            channel_version_map["esr"],
+            channel_version_map["esr_previous"],
+        }
+        for version in esr_versions:
+            channel = "esr"
+            flag = utils.get_flag(version, "status", channel)
+            active_versions.append((flag, channel, version))
+
+        return active_versions

bugbot/rules/regression_set_status_flags.py

@@ -2,9 +2,12 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from itertools import chain
+
 from libmozdata.bugzilla import Bugzilla
 
 from bugbot import logger, utils
+from bugbot.bug.analyzer import BugNotInStoreError, BugsStore
 from bugbot.bzcleaner import BzCleaner
 
 
@@ -204,6 +207,47 @@ class RegressionSetStatusFlags(BzCleaner):
                     info["esr"][f"esr{v}"] = "unaffected"
                 filtered_bugs[bugid] = info
 
+        # NOTE: The following is temporary to test that the new code is matching
+        # producing the expected results. Once we are confident with the new
+        # code, we can refactor this tool to use the new code.
+        # See: https://github.com/mozilla/bugbot/issues/2152
+        # >>>>> Begin of the temporary code
+        adjusted_bugs = (
+            {
+                **bug,
+                "regressed_by": [bug["regressed_by"]],
+            }
+            for bug in bugs.values()
+        )
+        bugs_store = BugsStore(chain(adjusted_bugs, data.values()))
+        for bug_id in bugs:
+            old_code_updates = self.status_changes.get(bug_id, {})
+            bug = bugs_store.get_bug_by_id(int(bug_id))
+            try:
+                new_code_updates = bug.detect_version_status_updates()
+            except BugNotInStoreError as error:
+                if self.dryrun:
+                    # This should be OK in local environment where we don't have
+                    # access to all bugs, but it is not OK in production.
+                    continue
+                raise error from None
+
+            new_code_updates = (
+                {update.flag: update.status for update in new_code_updates}
+                if new_code_updates
+                else {}
+            )
+
+            if old_code_updates != new_code_updates:
+                logger.error(
+                    "Rule %s: Mismatching status updates for bug %s: %s <--> %s",
+                    self.name(),
+                    bug_id,
+                    old_code_updates,
+                    new_code_updates,
+                )
+        # <<<<< End of the temporary code
+
         for bugid in filtered_bugs:
             regressor = bugs[bugid]["regressed_by"]
             self.status_changes[bugid]["comment"] = {

tests/test_bug_analyzer.py

@@ -0,0 +1,93 @@
+import unittest
+
+from bugbot.bug.analyzer import BugsStore, VersionStatus
+
+
+class TestSetStatusFlags(unittest.TestCase):
+    def test_set_status_flags(self):
+        all_bugs = [
+            {
+                "id": 1111,
+                "cf_status_firefox_esr2": "---",
+                "cf_status_firefox_esr3": "---",
+                "cf_status_firefox2": "---",
+                "cf_status_firefox3": "affected",
+                "cf_status_firefox4": "fixed",
+                "regressed_by": [111],
+            },
+            {
+                "id": 2222,
+                "cf_status_firefox_esr2": "---",
+                "cf_status_firefox_esr3": "---",
+                "cf_status_firefox2": "---",
+                "cf_status_firefox3": "---",
+                "cf_status_firefox4": "---",
+                "regressed_by": [222],
+            },
+            {
+                "id": 3333,
+                "cf_status_firefox_esr2": "---",
+                "cf_status_firefox_esr3": "---",
+                "cf_status_firefox2": "---",
+                "cf_status_firefox3": "affected",
+                "cf_status_firefox4": "fixed",
+                "regressed_by": [333],
+            },
+            {
+                "id": 111,
+                "cf_status_firefox_esr3": "fixed",
+                "cf_status_firefox3": "fixed",
+            },
+            {
+                "id": 222,
+                "cf_status_firefox1": "fixed",
+            },
+            {
+                "id": 333,
+                "cf_status_firefox_esr3": "fixed",
+                "cf_status_firefox3": "fixed",
+                "groups": ["core-security-release"],
+            },
+        ]
+
+        versions_map = {
+            "release": 2,
+            "beta": 3,
+            "nightly": 4,
+            "esr": 3,
+            "esr_previous": 2,
+        }
+
+        bugs_store = BugsStore(all_bugs, versions_map)
+
+        updates = bugs_store.get_bug_by_id(1111).detect_version_status_updates()
+        self.assertEqual(
+            updates,
+            [
+                VersionStatus(channel="release", version=2, status="unaffected"),
+                VersionStatus(channel="esr", version=2, status="unaffected"),
+                VersionStatus(channel="esr", version=3, status="affected"),
+            ],
+        )
+
+        updates = bugs_store.get_bug_by_id(2222).detect_version_status_updates()
+        self.assertEqual(
+            updates,
+            [
+                VersionStatus(channel="release", version=2, status="affected"),
+                VersionStatus(channel="beta", version=3, status="affected"),
+                VersionStatus(channel="nightly", version=4, status="affected"),
+                VersionStatus(channel="esr", version=2, status="affected"),
+                VersionStatus(channel="esr", version=3, status="affected"),
+            ],
+        )
+
+        updates = bugs_store.get_bug_by_id(3333).detect_version_status_updates()
+        self.assertEqual(
+            updates,
+            [
+                VersionStatus(channel="release", version=2, status="unaffected"),
+                VersionStatus(channel="esr", version=2, status="unaffected"),
+                VersionStatus(channel="esr", version=3, status="affected"),
+            ],
+        )

tests/test_regression_set_status_flags.py

@@ -3,12 +3,14 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 import unittest
 
+from libmozdata import versions as lmdversions
+
 from bugbot import utils
 from bugbot.bzcleaner import BzCleaner
 from bugbot.rules.regression_set_status_flags import RegressionSetStatusFlags
 
 
-def mock_get_checked_versions():
+def mock_get_checked_versions(base=True):
     return {
         "release": 2,
         "beta": 3,
@@ -75,11 +77,13 @@ def mock_get_flags_from_regressing_bugs(self, bugids):
 class TestSetStatusFlags(unittest.TestCase):
     def setUp(self):
         self.orig_get_checked_versions = utils.get_checked_versions
+        self.orig_get_versions = lmdversions.get
         self.orig_get_bugs = BzCleaner.get_bugs
         self.orig_get_flags_from_regressing_bugs = (
             RegressionSetStatusFlags.get_flags_from_regressing_bugs
         )
         utils.get_checked_versions = mock_get_checked_versions
+        lmdversions.get = mock_get_checked_versions
         BzCleaner.get_bugs = mock_get_bugs
         RegressionSetStatusFlags.get_flags_from_regressing_bugs = (
             mock_get_flags_from_regressing_bugs
@@ -87,6 +91,7 @@ class TestSetStatusFlags(unittest.TestCase):
 
     def tearDown(self):
         utils.get_checked_versions = self.orig_get_checked_versions
+        lmdversions.get = self.orig_get_versions
         BzCleaner.get_bugs = self.orig_get_bugs
         RegressionSetStatusFlags.get_flags_from_regressing_bugs = (
             self.orig_get_flags_from_regressing_bugs