diff --git a/build/mach_initialize.py b/build/mach_initialize.py index d6f054a3d212..e28068800966 100644 --- a/build/mach_initialize.py +++ b/build/mach_initialize.py @@ -503,12 +503,13 @@ def initialize(topsrcdir): # Sparse checkouts may not have all mach_commands.py files. Ignore # errors from missing files. Same for spidermonkey tarballs. repo = resolve_repository() - missing_ok = ( - repo is not None and repo.sparse_checkout_present() - ) or os.path.exists(os.path.join(topsrcdir, "INSTALL")) - + if repo != "SOURCE": + missing_ok = ( + repo is not None and repo.sparse_checkout_present() + ) or os.path.exists(os.path.join(topsrcdir, "INSTALL")) + else: + missing_ok = () driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok) - return driver diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py index 540722c9104b..c32946c4ebd0 100644 --- a/python/mozboot/mozboot/base.py +++ b/python/mozboot/mozboot/base.py @@ -349,7 +349,10 @@ class BaseBootstrapper(object): def install_toolchain_artifact_impl( self, install_dir: Path, toolchain_job, no_unpack=False ): - mach_binary = (self.srcdir / "mach").resolve() + if type(self.srcdir) is str: + mach_binary = Path(self.srcdir) / "mach" + else: + mach_binary = (self.srcdir / "mach").resolve() if not mach_binary.exists(): raise ValueError(f"mach not found at {mach_binary}") diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py index 2ab5293f2ed0..f5c2bd047bfc 100644 --- a/python/mozboot/mozboot/bootstrap.py +++ b/python/mozboot/mozboot/bootstrap.py @@ -253,7 +253,7 @@ class Bootstrapper(object): self.instance = cls(**args) - def maybe_install_private_packages_or_exit(self, application): + def maybe_install_private_packages_or_exit(self, application, checkout_type): # Install the clang packages needed for building the style system, as # well as the version of NodeJS that we currently support. # Also install the clang static-analysis package by default @@ -345,7 +345,7 @@ class Bootstrapper(object): self._validate_python_environment(checkout_root) if self.instance.no_system_changes: - self.maybe_install_private_packages_or_exit(application) + self.maybe_install_private_packages_or_exit(application, checkout_type) self._output_mozconfig(application, mozconfig_builder) sys.exit(0) @@ -391,7 +391,7 @@ class Bootstrapper(object): checkout_root, ) - self.maybe_install_private_packages_or_exit(application) + self.maybe_install_private_packages_or_exit(application, checkout_type) self.check_code_submission(checkout_root) # Wait until after moz-phab setup to check telemetry so that employees # will be automatically opted-in. @@ -600,6 +600,7 @@ def current_firefox_checkout(env, hg: Optional[Path] = None): while path: hg_dir = path / ".hg" git_dir = path / ".git" + moz_configure = path / "moz.configure" if hg and hg_dir.exists(): # Verify the hg repo is a Firefox repo by looking at rev 0. try: @@ -625,6 +626,8 @@ def current_firefox_checkout(env, hg: Optional[Path] = None): if moz_configure.exists(): _warn_if_risky_revision(path) return ("git" if git_dir.exists() else "hg"), path + elif moz_configure.exists(): + return "SOURCE", path if not len(path.parents): break diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py index e1e3b3134f01..df736e5d84bb 100644 --- a/python/mozbuild/mozbuild/artifact_commands.py +++ b/python/mozbuild/mozbuild/artifact_commands.py @@ -2,6 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + import argparse import hashlib import json @@ -183,7 +185,12 @@ def artifact_install( no_process=no_process, ) - return artifacts.install_from(source, distdir or command_context.distdir) + if source is None and distdir is None: + return artifacts.install_from( + source, distdir or command_context.distdir, src=True + ) + else: + return artifacts.install_from(source, distdir or command_context.distdir) @ArtifactSubCommand( @@ -433,17 +440,18 @@ def artifact_toolchain( repo = mozversioncontrol.get_repository_object( command_context.topsrcdir ) - changed_files = set(repo.get_outgoing_files()) | set( - repo.get_changed_files() - ) - if changed_files: - command_context.log( - logging.ERROR, - "artifact", - {}, - "Hint: consider reverting your local changes " - "to the following files: %s" % sorted(changed_files), + if not isinstance(repo, mozversioncontrol.SrcRepository): + changed_files = set(repo.get_outgoing_files()) | set( + repo.get_changed_files() ) + if changed_files: + command_context.log( + logging.ERROR, + "artifact", + {}, + "Hint: consider reverting your local changes " + "to the following files: %s" % sorted(changed_files), + ) if "TASKCLUSTER_ROOT_URL" in os.environ: command_context.log( logging.ERROR, diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py index 87f96be70434..9822a9b76e53 100644 --- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -482,7 +482,12 @@ class MozbuildObject(ProcessExecutionMixin): except InvalidRepoPath: repo = None - if repo and not vcs_revision and repo.sparse_checkout_present(): + if ( + repo + and repo != "SOURCE" + and not vcs_revision + and repo.sparse_checkout_present() + ): vcs_revision = "." if vcs_revision is None: diff --git a/python/mozversioncontrol/mozversioncontrol/__init__.py b/python/mozversioncontrol/mozversioncontrol/__init__.py index 70dbac4032c2..879e462b3fca 100644 --- a/python/mozversioncontrol/mozversioncontrol/__init__.py +++ b/python/mozversioncontrol/mozversioncontrol/__init__.py @@ -50,7 +50,7 @@ class CannotDeleteFromRootOfRepositoryException(Exception): the repository, which is not permitted.""" -def get_tool_path(tool: Union[str, Path]): +def get_tool_path(tool: Optional[Union[str, Path]] = None): tool = Path(tool) """Obtain the path of `tool`.""" if tool.is_absolute() and tool.exists(): @@ -78,9 +78,9 @@ class Repository(object): __metaclass__ = abc.ABCMeta - def __init__(self, path: Path, tool: str): + def __init__(self, path: Path, tool: Optional[str] = None): self.path = str(path.resolve()) - self._tool = Path(get_tool_path(tool)) + self._tool = Path(get_tool_path(tool)) if tool else None self._version = None self._valid_diff_filter = ("m", "a", "d") self._env = os.environ.copy() @@ -206,7 +206,7 @@ class Repository(object): """Undo the effects of a previous add_remove_files call for `paths`.""" @abc.abstractmethod - def get_tracked_files_finder(self): + def get_tracked_files_finder(self, path=None): """Obtain a mozpack.files.BaseFinder of managed files in the working directory. @@ -495,7 +495,7 @@ class HgRepository(Repository): self._run("forget", *paths) - def get_tracked_files_finder(self): + def get_tracked_files_finder(self, path=None): # Can return backslashes on Windows. Normalize to forward slashes. files = list( p.replace("\\", "/") for p in self._run("files", "-0").split("\0") if p @@ -672,7 +672,7 @@ class GitRepository(Repository): self._run("reset", *paths) - def get_tracked_files_finder(self): + def get_tracked_files_finder(self, path=None): files = [p for p in self._run("ls-files", "-z").split("\0") if p] return FileListFinder(files) @@ -744,7 +744,130 @@ class GitRepository(Repository): self._run("config", name, value) -def get_repository_object(path: Optional[Union[str, Path]], hg="hg", git="git"): +class SrcRepository(Repository): + """An implementation of `Repository` for Git repositories.""" + + def __init__(self, path: Path, src="src"): + super(SrcRepository, self).__init__(path, tool=None) + + @property + def name(self): + return "src" + + @property + def head_ref(self): + pass + + @property + def base_ref(self): + pass + + def base_ref_as_hg(self): + pass + + @property + def branch(self): + pass + + @property + def has_git_cinnabar(self): + pass + + def get_commit_time(self): + pass + + def sparse_checkout_present(self): + pass + + def get_user_email(self): + pass + + def get_upstream(self): + pass + + def get_changed_files(self, diff_filter="ADM", mode="unstaged", rev=None): + pass + + def get_outgoing_files(self, diff_filter="ADM", upstream=None): + pass + + def add_remove_files(self, *paths: Union[str, Path]): + pass + + def forget_add_remove_files(self, *paths: Union[str, Path]): + pass + + def git_ignore(self, path): + """This function reads the mozilla-central/.gitignore file and creates a + list of the patterns to ignore + """ + ignore = [] + f = open(path + "/.gitignore", "r") + while True: + line = f.readline() + if not line: + break + if line.startswith("#"): + pass + elif line.strip() and line not in ["\r", "\r\n"]: + ignore.append(line.strip().lstrip("/")) + f.close() + return ignore + + def get_files(self, path): + """This function gets all files in your source folder e.g mozilla-central + and creates a list of that + """ + res = [] + # move away the .git or .hg folder from path to more easily test in a hg/git repo + for root, dirs, files in os.walk("."): + for name in files: + res.append(os.path.join(root, name)) + return res + + def get_tracked_files_finder(self, path): + """Get files, similar to 'hg files -0' or 'git ls-files -z', thats why + we read the .gitignore file for patterns to ignore. + Speed could probably be improved. + """ + import fnmatch + + files = list( + p.replace("\\", "/").replace("./", "") for p in self.get_files(path) if p + ) + files.sort() + ig = self.git_ignore(path) + mat = [] + for i in ig: + x = fnmatch.filter(files, i) + if x: + mat = mat + x + match = list(set(files) - set(mat)) + match.sort() + if len(match) == 0: + return None + else: + return FileListFinder(match) + + def working_directory_clean(self, untracked=False, ignored=False): + pass + + def clean_directory(self, path: Union[str, Path]): + pass + + def update(self, ref): + pass + + def push_to_try(self, message, allow_log_capture=False): + pass + + def set_config(self, name, value): + pass + + +def get_repository_object( + path: Optional[Union[str, Path]], hg="hg", git="git", src="src" +): """Get a repository object for the repository at `path`. If `path` is not a known VCS repository, raise an exception. """ @@ -756,6 +879,8 @@ def get_repository_object(path: Optional[Union[str, Path]], hg="hg", git="git"): return HgRepository(path, hg=hg) elif (path / ".git").exists(): return GitRepository(path, git=git) + elif (path / "moz.configure").exists(): + return SrcRepository(path, src=src) else: raise InvalidRepoPath(f"Unknown VCS, or not a source checkout: {path}") @@ -781,6 +906,8 @@ def get_repository_from_build_config(config): return HgRepository(Path(config.topsrcdir), hg=config.substs["HG"]) elif flavor == "git": return GitRepository(Path(config.topsrcdir), git=config.substs["GIT"]) + elif flavor == "src": + return SrcRepository(Path(config.topsrcdir), src=config.substs["SRC"]) else: raise MissingVCSInfo("unknown VCS_CHECKOUT_TYPE value: %s" % flavor) diff --git a/taskcluster/gecko_taskgraph/util/hash.py b/taskcluster/gecko_taskgraph/util/hash.py index 49b582f6d47d..485c9a7c480e 100644 --- a/taskcluster/gecko_taskgraph/util/hash.py +++ b/taskcluster/gecko_taskgraph/util/hash.py @@ -21,7 +21,19 @@ def hash_path(path): @memoize def get_file_finder(base_path): - return get_repository_object(base_path).get_tracked_files_finder() + from pathlib import Path + + repo = get_repository_object(base_path) + if repo: + files = repo.get_tracked_files_finder(base_path) + if files: + return files + else: + return None + else: + return get_repository_object(Path(base_path)).get_tracked_files_finder( + base_path + ) def hash_paths(base_path, patterns): @@ -36,19 +48,21 @@ def hash_paths(base_path, patterns): finder = get_file_finder(base_path) h = hashlib.sha256() files = {} - for pattern in patterns: - found = list(finder.find(pattern)) - if found: - files.update(found) - else: - raise Exception("%s did not match anything" % pattern) - for path in sorted(files.keys()): - if path.endswith((".pyc", ".pyd", ".pyo")): - continue - h.update( - "{} {}\n".format( - hash_path(mozpath.abspath(mozpath.join(base_path, path))), - mozpath.normsep(path), - ).encode("utf-8") - ) + if finder: + for pattern in patterns: + found = list(finder.find(pattern)) + if found: + files.update(found) + else: + raise Exception("%s did not match anything" % pattern) + for path in sorted(files.keys()): + if path.endswith((".pyc", ".pyd", ".pyo")): + continue + h.update( + "{} {}\n".format( + hash_path(mozpath.abspath(mozpath.join(base_path, path))), + mozpath.normsep(path), + ).encode("utf-8") + ) + return h.hexdigest() diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/generator.py b/third_party/python/taskcluster_taskgraph/taskgraph/generator.py index e1b900cf658e..9af19bc2ebbb 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/generator.py +++ b/third_party/python/taskcluster_taskgraph/taskgraph/generator.py @@ -271,7 +271,7 @@ class TaskGraphGenerator: # Always add legacy target tasks method until we deprecate that API. if "target_tasks_method" not in filters: filters.insert(0, "target_tasks_method") - filters = [filter_tasks.filter_task_functions[f] for f in filters] + filters = [filter_tasks.filter_task_functions[f] for f in filters if f] yield self.verify("parameters", parameters) diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/parameters.py b/third_party/python/taskcluster_taskgraph/taskgraph/parameters.py index ed662c704e4c..276ce4ffc711 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/parameters.py +++ b/third_party/python/taskcluster_taskgraph/taskgraph/parameters.py @@ -80,46 +80,76 @@ def get_version(repo_path): def _get_defaults(repo_root=None): repo_path = repo_root or os.getcwd() repo = get_repository(repo_path) - try: - repo_url = repo.get_url() - parsed_url = mozilla_repo_urls.parse(repo_url) - project = parsed_url.repo_name - except ( - CalledProcessError, - mozilla_repo_urls.errors.InvalidRepoUrlError, - mozilla_repo_urls.errors.UnsupportedPlatformError, - ): - repo_url = "" - project = "" + if repo: + try: + repo_url = repo.get_url() + parsed_url = mozilla_repo_urls.parse(repo_url) + project = parsed_url.repo_name + except ( + CalledProcessError, + mozilla_repo_urls.errors.InvalidRepoUrlError, + mozilla_repo_urls.errors.UnsupportedPlatformError, + ): + repo_url = "" + project = "" - return { - "base_repository": repo_url, - "base_ref": "", - "base_rev": "", - "build_date": int(time.time()), - "build_number": 1, - "do_not_optimize": [], - "enable_always_target": True, - "existing_tasks": {}, - "filters": ["target_tasks_method"], - "head_ref": repo.branch or repo.head_rev, - "head_repository": repo_url, - "head_rev": repo.head_rev, - "head_tag": "", - "level": "3", - "moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"), - "next_version": None, - "optimize_strategies": None, - "optimize_target_tasks": True, - "owner": "nobody@mozilla.com", - "project": project, - "pushdate": int(time.time()), - "pushlog_id": "0", - "repository_type": repo.tool, - "target_tasks_method": "default", - "tasks_for": "", - "version": get_version(repo_path), - } + return { + "base_repository": repo_url, + "base_ref": "", + "base_rev": "", + "build_date": int(time.time()), + "build_number": 1, + "do_not_optimize": [], + "enable_always_target": True, + "existing_tasks": {}, + "filters": ["target_tasks_method"], + "head_ref": repo.branch or repo.head_rev, + "head_repository": repo_url, + "head_rev": repo.head_rev, + "head_tag": "", + "level": "3", + "moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"), + "next_version": None, + "optimize_strategies": None, + "optimize_target_tasks": True, + "owner": "nobody@mozilla.com", + "project": project, + "pushdate": int(time.time()), + "pushlog_id": "0", + "repository_type": repo.tool, + "target_tasks_method": "default", + "tasks_for": "", + "version": get_version(repo_path), + } + else: + return { + "base_repository": "SOURCE", + "base_ref": "", + "base_rev": "", + "build_date": int(time.time()), + "build_number": 1, + "do_not_optimize": [], + "enable_always_target": True, + "existing_tasks": {}, + "filters": ["target_tasks_method"], + "head_ref": "", + "head_repository": "", + "head_rev": "", + "head_tag": "", + "level": "3", + "moz_build_date": datetime.now().strftime("%Y%m%d%H%M%S"), + "next_version": None, + "optimize_strategies": None, + "optimize_target_tasks": True, + "owner": "nobody@mozilla.com", + "project": "", + "pushdate": int(time.time()), + "pushlog_id": "0", + "repository_type": "", + "target_tasks_method": "default", + "tasks_for": "", + "version": "", + } defaults_functions = [_get_defaults] @@ -195,13 +225,14 @@ class Parameters(ReadOnlyDict): @staticmethod def _fill_defaults(repo_root=None, **kwargs): defaults = {} - for fn in defaults_functions: - defaults.update(fn(repo_root)) + if repo_root != "SOURCE": + for fn in defaults_functions: + defaults.update(fn(repo_root)) - for name, default in defaults.items(): - if name not in kwargs: - kwargs[name] = default - return kwargs + for name, default in defaults.items(): + if name not in kwargs: + kwargs[name] = default + return kwargs def check(self): schema = ( diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py b/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py index ba1d90901991..30744fe3f4d8 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py +++ b/third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py @@ -504,8 +504,10 @@ def get_repository(path): return HgRepository(path) elif os.path.exists(os.path.join(path, ".git")): return GitRepository(path) + elif os.path.exists(os.path.join(path, "moz.configure")): + return None - raise RuntimeError("Current directory is neither a git or hg repository") + raise RuntimeError("Current directory is neither a git or hg repository, nor a release source") def find_hg_revision_push_info(repository, revision):