Bug 1798746 - Ensure checking for mercurial/git only if its a hg/git clone, not release source r=ahochheiden

Differential Revision: https://phabricator.services.mozilla.com/D162461

build/mach_initialize.py

@@ -503,12 +503,13 @@ def initialize(topsrcdir):
     # Sparse checkouts may not have all mach_commands.py files. Ignore
     # errors from missing files. Same for spidermonkey tarballs.
     repo = resolve_repository()
-    missing_ok = (
-        repo is not None and repo.sparse_checkout_present()
-    ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
-
+    if repo != "SOURCE":
+        missing_ok = (
+            repo is not None and repo.sparse_checkout_present()
+        ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
+    else:
+        missing_ok = ()
     driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok)
-
     return driver
 
 

python/mozboot/mozboot/base.py

@@ -349,7 +349,10 @@ class BaseBootstrapper(object):
     def install_toolchain_artifact_impl(
         self, install_dir: Path, toolchain_job, no_unpack=False
     ):
-        mach_binary = (self.srcdir / "mach").resolve()
+        if type(self.srcdir) is str:
+            mach_binary = Path(self.srcdir) / "mach"
+        else:
+            mach_binary = (self.srcdir / "mach").resolve()
         if not mach_binary.exists():
             raise ValueError(f"mach not found at {mach_binary}")
 

python/mozboot/mozboot/bootstrap.py

@@ -253,7 +253,7 @@ class Bootstrapper(object):
 
         self.instance = cls(**args)
 
-    def maybe_install_private_packages_or_exit(self, application):
+    def maybe_install_private_packages_or_exit(self, application, checkout_type):
         # Install the clang packages needed for building the style system, as
         # well as the version of NodeJS that we currently support.
         # Also install the clang static-analysis package by default
@@ -345,7 +345,7 @@ class Bootstrapper(object):
         self._validate_python_environment(checkout_root)
 
         if self.instance.no_system_changes:
-            self.maybe_install_private_packages_or_exit(application)
+            self.maybe_install_private_packages_or_exit(application, checkout_type)
             self._output_mozconfig(application, mozconfig_builder)
             sys.exit(0)
 
@@ -391,7 +391,7 @@ class Bootstrapper(object):
                     checkout_root,
                 )
 
-        self.maybe_install_private_packages_or_exit(application)
+        self.maybe_install_private_packages_or_exit(application, checkout_type)
         self.check_code_submission(checkout_root)
         # Wait until after moz-phab setup to check telemetry so that employees
         # will be automatically opted-in.
@@ -600,6 +600,7 @@ def current_firefox_checkout(env, hg: Optional[Path] = None):
     while path:
         hg_dir = path / ".hg"
         git_dir = path / ".git"
+        moz_configure = path / "moz.configure"
         if hg and hg_dir.exists():
             # Verify the hg repo is a Firefox repo by looking at rev 0.
             try:
@@ -625,6 +626,8 @@ def current_firefox_checkout(env, hg: Optional[Path] = None):
             if moz_configure.exists():
                 _warn_if_risky_revision(path)
                 return ("git" if git_dir.exists() else "hg"), path
+        elif moz_configure.exists():
+            return "SOURCE", path
 
         if not len(path.parents):
             break

python/mozbuild/mozbuild/artifact_commands.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import argparse
 import hashlib
 import json
@@ -183,7 +185,12 @@ def artifact_install(
         no_process=no_process,
     )
 
-    return artifacts.install_from(source, distdir or command_context.distdir)
+    if source is None and distdir is None:
+        return artifacts.install_from(
+            source, distdir or command_context.distdir, src=True
+        )
+    else:
+        return artifacts.install_from(source, distdir or command_context.distdir)
 
 
 @ArtifactSubCommand(
@@ -433,17 +440,18 @@ def artifact_toolchain(
                 repo = mozversioncontrol.get_repository_object(
                     command_context.topsrcdir
                 )
-                changed_files = set(repo.get_outgoing_files()) | set(
-                    repo.get_changed_files()
-                )
-                if changed_files:
-                    command_context.log(
-                        logging.ERROR,
-                        "artifact",
-                        {},
-                        "Hint: consider reverting your local changes "
-                        "to the following files: %s" % sorted(changed_files),
+                if not isinstance(repo, mozversioncontrol.SrcRepository):
+                    changed_files = set(repo.get_outgoing_files()) | set(
+                        repo.get_changed_files()
                     )
+                    if changed_files:
+                        command_context.log(
+                            logging.ERROR,
+                            "artifact",
+                            {},
+                            "Hint: consider reverting your local changes "
+                            "to the following files: %s" % sorted(changed_files),
+                        )
                 if "TASKCLUSTER_ROOT_URL" in os.environ:
                     command_context.log(
                         logging.ERROR,

python/mozbuild/mozbuild/base.py

@@ -482,7 +482,12 @@ class MozbuildObject(ProcessExecutionMixin):
         except InvalidRepoPath:
             repo = None
 
-        if repo and not vcs_revision and repo.sparse_checkout_present():
+        if (
+            repo
+            and repo != "SOURCE"
+            and not vcs_revision
+            and repo.sparse_checkout_present()
+        ):
             vcs_revision = "."
 
         if vcs_revision is None:

python/mozversioncontrol/mozversioncontrol/__init__.py

@@ -50,7 +50,7 @@ class CannotDeleteFromRootOfRepositoryException(Exception):
     the repository, which is not permitted."""
 
 
-def get_tool_path(tool: Union[str, Path]):
+def get_tool_path(tool: Optional[Union[str, Path]] = None):
     tool = Path(tool)
     """Obtain the path of `tool`."""
     if tool.is_absolute() and tool.exists():
@@ -78,9 +78,9 @@ class Repository(object):
 
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, path: Path, tool: str):
+    def __init__(self, path: Path, tool: Optional[str] = None):
         self.path = str(path.resolve())
-        self._tool = Path(get_tool_path(tool))
+        self._tool = Path(get_tool_path(tool)) if tool else None
         self._version = None
         self._valid_diff_filter = ("m", "a", "d")
         self._env = os.environ.copy()
@@ -206,7 +206,7 @@ class Repository(object):
         """Undo the effects of a previous add_remove_files call for `paths`."""
 
     @abc.abstractmethod
-    def get_tracked_files_finder(self):
+    def get_tracked_files_finder(self, path=None):
         """Obtain a mozpack.files.BaseFinder of managed files in the working
         directory.
 
@@ -495,7 +495,7 @@ class HgRepository(Repository):
 
         self._run("forget", *paths)
 
-    def get_tracked_files_finder(self):
+    def get_tracked_files_finder(self, path=None):
         # Can return backslashes on Windows. Normalize to forward slashes.
         files = list(
             p.replace("\\", "/") for p in self._run("files", "-0").split("\0") if p
@@ -672,7 +672,7 @@ class GitRepository(Repository):
 
         self._run("reset", *paths)
 
-    def get_tracked_files_finder(self):
+    def get_tracked_files_finder(self, path=None):
         files = [p for p in self._run("ls-files", "-z").split("\0") if p]
         return FileListFinder(files)
 
@@ -744,7 +744,130 @@ class GitRepository(Repository):
         self._run("config", name, value)
 
 
-def get_repository_object(path: Optional[Union[str, Path]], hg="hg", git="git"):
+class SrcRepository(Repository):
+    """An implementation of `Repository` for Git repositories."""
+
+    def __init__(self, path: Path, src="src"):
+        super(SrcRepository, self).__init__(path, tool=None)
+
+    @property
+    def name(self):
+        return "src"
+
+    @property
+    def head_ref(self):
+        pass
+
+    @property
+    def base_ref(self):
+        pass
+
+    def base_ref_as_hg(self):
+        pass
+
+    @property
+    def branch(self):
+        pass
+
+    @property
+    def has_git_cinnabar(self):
+        pass
+
+    def get_commit_time(self):
+        pass
+
+    def sparse_checkout_present(self):
+        pass
+
+    def get_user_email(self):
+        pass
+
+    def get_upstream(self):
+        pass
+
+    def get_changed_files(self, diff_filter="ADM", mode="unstaged", rev=None):
+        pass
+
+    def get_outgoing_files(self, diff_filter="ADM", upstream=None):
+        pass
+
+    def add_remove_files(self, *paths: Union[str, Path]):
+        pass
+
+    def forget_add_remove_files(self, *paths: Union[str, Path]):
+        pass
+
+    def git_ignore(self, path):
+        """This function reads the mozilla-central/.gitignore file and creates a
+        list of the patterns to ignore
+        """
+        ignore = []
+        f = open(path + "/.gitignore", "r")
+        while True:
+            line = f.readline()
+            if not line:
+                break
+            if line.startswith("#"):
+                pass
+            elif line.strip() and line not in ["\r", "\r\n"]:
+                ignore.append(line.strip().lstrip("/"))
+        f.close()
+        return ignore
+
+    def get_files(self, path):
+        """This function gets all files in your source folder e.g mozilla-central
+        and creates a list of that
+        """
+        res = []
+        # move away the .git or .hg folder from path to more easily test in a hg/git repo
+        for root, dirs, files in os.walk("."):
+            for name in files:
+                res.append(os.path.join(root, name))
+        return res
+
+    def get_tracked_files_finder(self, path):
+        """Get files, similar to 'hg files -0' or 'git ls-files -z', thats why
+        we read the .gitignore file for patterns to ignore.
+        Speed could probably be improved.
+        """
+        import fnmatch
+
+        files = list(
+            p.replace("\\", "/").replace("./", "") for p in self.get_files(path) if p
+        )
+        files.sort()
+        ig = self.git_ignore(path)
+        mat = []
+        for i in ig:
+            x = fnmatch.filter(files, i)
+            if x:
+                mat = mat + x
+        match = list(set(files) - set(mat))
+        match.sort()
+        if len(match) == 0:
+            return None
+        else:
+            return FileListFinder(match)
+
+    def working_directory_clean(self, untracked=False, ignored=False):
+        pass
+
+    def clean_directory(self, path: Union[str, Path]):
+        pass
+
+    def update(self, ref):
+        pass
+
+    def push_to_try(self, message, allow_log_capture=False):
+        pass
+
+    def set_config(self, name, value):
+        pass
+
+
+def get_repository_object(
+    path: Optional[Union[str, Path]], hg="hg", git="git", src="src"
+):
     """Get a repository object for the repository at `path`.
     If `path` is not a known VCS repository, raise an exception.
     """
@@ -756,6 +879,8 @@ def get_repository_object(path: Optional[Union[str, Path]], hg="hg", git="git"):
         return HgRepository(path, hg=hg)
     elif (path / ".git").exists():
         return GitRepository(path, git=git)
+    elif (path / "moz.configure").exists():
+        return SrcRepository(path, src=src)
     else:
         raise InvalidRepoPath(f"Unknown VCS, or not a source checkout: {path}")
 
@@ -781,6 +906,8 @@ def get_repository_from_build_config(config):
         return HgRepository(Path(config.topsrcdir), hg=config.substs["HG"])
     elif flavor == "git":
         return GitRepository(Path(config.topsrcdir), git=config.substs["GIT"])
+    elif flavor == "src":
+        return SrcRepository(Path(config.topsrcdir), src=config.substs["SRC"])
     else:
         raise MissingVCSInfo("unknown VCS_CHECKOUT_TYPE value: %s" % flavor)
 

taskcluster/gecko_taskgraph/util/hash.py

@@ -21,7 +21,19 @@ def hash_path(path):
 
 @memoize
 def get_file_finder(base_path):
-    return get_repository_object(base_path).get_tracked_files_finder()
+    from pathlib import Path
+
+    repo = get_repository_object(base_path)
+    if repo:
+        files = repo.get_tracked_files_finder(base_path)
+        if files:
+            return files
+        else:
+            return None
+    else:
+        return get_repository_object(Path(base_path)).get_tracked_files_finder(
+            base_path
+        )
 
 
 def hash_paths(base_path, patterns):
@@ -36,19 +48,21 @@ def hash_paths(base_path, patterns):
     finder = get_file_finder(base_path)
     h = hashlib.sha256()
     files = {}
-    for pattern in patterns:
-        found = list(finder.find(pattern))
-        if found:
-            files.update(found)
-        else:
-            raise Exception("%s did not match anything" % pattern)
-    for path in sorted(files.keys()):
-        if path.endswith((".pyc", ".pyd", ".pyo")):
-            continue
-        h.update(
-            "{} {}\n".format(
-                hash_path(mozpath.abspath(mozpath.join(base_path, path))),
-                mozpath.normsep(path),
-            ).encode("utf-8")
-        )
+    if finder:
+        for pattern in patterns:
+            found = list(finder.find(pattern))
+            if found:
+                files.update(found)
+            else:
+                raise Exception("%s did not match anything" % pattern)
+        for path in sorted(files.keys()):
+            if path.endswith((".pyc", ".pyd", ".pyo")):
+                continue
+            h.update(
+                "{} {}\n".format(
+                    hash_path(mozpath.abspath(mozpath.join(base_path, path))),
+                    mozpath.normsep(path),
+                ).encode("utf-8")
+            )
+
     return h.hexdigest()

third_party/python/taskcluster_taskgraph/taskgraph/generator.py

@@ -271,7 +271,7 @@ class TaskGraphGenerator:
         # Always add legacy target tasks method until we deprecate that API.
         if "target_tasks_method" not in filters:
             filters.insert(0, "target_tasks_method")
-        filters = [filter_tasks.filter_task_functions[f] for f in filters]
+        filters = [filter_tasks.filter_task_functions[f] for f in filters if f]
 
         yield self.verify("parameters", parameters)
 

third_party/python/taskcluster_taskgraph/taskgraph/parameters.py

@@ -80,46 +80,76 @@ def get_version(repo_path):
 def _get_defaults(repo_root=None):
     repo_path = repo_root or os.getcwd()
     repo = get_repository(repo_path)
-    try:
-        repo_url = repo.get_url()
-        parsed_url = mozilla_repo_urls.parse(repo_url)
-        project = parsed_url.repo_name
-    except (
-        CalledProcessError,
-        mozilla_repo_urls.errors.InvalidRepoUrlError,
-        mozilla_repo_urls.errors.UnsupportedPlatformError,
-    ):
-        repo_url = ""
-        project = ""
+    if repo:
+        try:
+            repo_url = repo.get_url()
+            parsed_url = mozilla_repo_urls.parse(repo_url)
+            project = parsed_url.repo_name
+        except (
+            CalledProcessError,
+            mozilla_repo_urls.errors.InvalidRepoUrlError,
+            mozilla_repo_urls.errors.UnsupportedPlatformError,
+        ):
+            repo_url = ""
+            project = ""
 
-    return {
-        "base_repository": repo_url,
-        "base_ref": "",
-        "base_rev": "",
-        "build_date": int(time.time()),
-        "build_number": 1,
-        "do_not_optimize": [],
-        "enable_always_target": True,
-        "existing_tasks": {},
-        "filters": ["target_tasks_method"],
-        "head_ref": repo.branch or repo.head_rev,
-        "head_repository": repo_url,
-        "head_rev": repo.head_rev,
-        "head_tag": "",
-        "level": "3",
-        "moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"),
-        "next_version": None,
-        "optimize_strategies": None,
-        "optimize_target_tasks": True,
-        "owner": "nobody@mozilla.com",
-        "project": project,
-        "pushdate": int(time.time()),
-        "pushlog_id": "0",
-        "repository_type": repo.tool,
-        "target_tasks_method": "default",
-        "tasks_for": "",
-        "version": get_version(repo_path),
-    }
+        return {
+            "base_repository": repo_url,
+            "base_ref": "",
+            "base_rev": "",
+            "build_date": int(time.time()),
+            "build_number": 1,
+            "do_not_optimize": [],
+            "enable_always_target": True,
+            "existing_tasks": {},
+            "filters": ["target_tasks_method"],
+            "head_ref": repo.branch or repo.head_rev,
+            "head_repository": repo_url,
+            "head_rev": repo.head_rev,
+            "head_tag": "",
+            "level": "3",
+            "moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"),
+            "next_version": None,
+            "optimize_strategies": None,
+            "optimize_target_tasks": True,
+            "owner": "nobody@mozilla.com",
+            "project": project,
+            "pushdate": int(time.time()),
+            "pushlog_id": "0",
+            "repository_type": repo.tool,
+            "target_tasks_method": "default",
+            "tasks_for": "",
+            "version": get_version(repo_path),
+        }
+    else:
+        return {
+            "base_repository": "SOURCE",
+            "base_ref": "",
+            "base_rev": "",
+            "build_date": int(time.time()),
+            "build_number": 1,
+            "do_not_optimize": [],
+            "enable_always_target": True,
+            "existing_tasks": {},
+            "filters": ["target_tasks_method"],
+            "head_ref": "",
+            "head_repository": "",
+            "head_rev": "",
+            "head_tag": "",
+            "level": "3",
+            "moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"),
+            "next_version": None,
+            "optimize_strategies": None,
+            "optimize_target_tasks": True,
+            "owner": "nobody@mozilla.com",
+            "project": "",
+            "pushdate": int(time.time()),
+            "pushlog_id": "0",
+            "repository_type": "",
+            "target_tasks_method": "default",
+            "tasks_for": "",
+            "version": "",
+        }
 
 
 defaults_functions = [_get_defaults]
@@ -195,13 +225,14 @@ class Parameters(ReadOnlyDict):
     @staticmethod
     def _fill_defaults(repo_root=None, **kwargs):
         defaults = {}
-        for fn in defaults_functions:
-            defaults.update(fn(repo_root))
+        if repo_root != "SOURCE":
+            for fn in defaults_functions:
+                defaults.update(fn(repo_root))
 
-        for name, default in defaults.items():
-            if name not in kwargs:
-                kwargs[name] = default
-        return kwargs
+            for name, default in defaults.items():
+                if name not in kwargs:
+                    kwargs[name] = default
+            return kwargs
 
     def check(self):
         schema = (

third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py

@@ -504,8 +504,10 @@ def get_repository(path):
             return HgRepository(path)
         elif os.path.exists(os.path.join(path, ".git")):
             return GitRepository(path)
+        elif os.path.exists(os.path.join(path, "moz.configure")):
+            return None
 
-    raise RuntimeError("Current directory is neither a git or hg repository")
+    raise RuntimeError("Current directory is neither a git or hg repository, nor a release source")
 
 
 def find_hg_revision_push_info(repository, revision):