diff --git a/comma/downstream/monitor.py b/comma/downstream/monitor.py index 62e1fec..769a0ee 100755 --- a/comma/downstream/monitor.py +++ b/comma/downstream/monitor.py @@ -6,9 +6,6 @@ Functions for monitoring downstream repos for missing commits import logging -import approxidate -import git - from comma.database.driver import DatabaseDriver from comma.database.model import ( Distros, @@ -18,8 +15,7 @@ from comma.database.model import ( ) from comma.downstream.matcher import patch_matches from comma.upstream import process_commits -from comma.util import config -from comma.util.tracking import GitProgressPrinter, Repo, get_linux_repo +from comma.util.tracking import get_linux_repo LOGGER = logging.getLogger(__name__) @@ -70,49 +66,14 @@ def update_tracked_revisions(distro_id, repo): # ls-remote could naturally sort by date, but that would require all the objects to be local if distro_id.startswith("Ubuntu"): - tag_names = [] - for line in repo.git.ls_remote( - "--tags", "--refs", "--sort=v:refname", distro_id - ).splitlines(): - name = line.split("/", 1)[-1] - if "azure" in name and all(label not in name for label in ("edge", "cvm", "fde")): - tag_names.append(name) - + tag_names = tuple( + tag + for tag in repo.get_remote_tags(distro_id) + if "azure" in tag and all(label not in tag for label in ("edge", "cvm", "fde")) + ) update_revisions_for_distro(distro_id, tag_names[-2:]) -def get_missing_cherries(repo, reference): - """ - Get a list of cherry-picked commits missing from the downstream reference - """ - - # Get all upstream commits on tracked paths within window - upstream_commits = set( - repo.git.log( - "--no-merges", - "--pretty=format:%H", - f"--since={config.since}", - "origin/master", - "--", - repo.get_tracked_paths(), - ).splitlines() - ) - - # Get missing cherries for all paths, but don't filter by path since it takes forever - missing_cherries = set( - repo.git.log( - "--no-merges", - "--right-only", - "--cherry-pick", - "--pretty=format:%H", - f"--since={config.since}", - f"{reference}...origin/master", - ).splitlines() - ) - - return missing_cherries & upstream_commits - - def monitor_subject(monitoring_subject, repo, reference=None): """ Update the missing patches in the database for this monitoring_subject @@ -123,7 +84,7 @@ def monitor_subject(monitoring_subject, repo, reference=None): reference = monitoring_subject.revision if reference is None else reference - missing_cherries = get_missing_cherries(repo, reference) + missing_cherries = repo.get_missing_cherries(reference, repo.get_tracked_paths()) LOGGER.debug("Found %d missing patches through cherry-pick.", len(missing_cherries)) # Run extra checks on these missing commits @@ -193,56 +154,6 @@ def get_missing_patch_ids(missing_cherries, reference): return missing_patches -def fetch_remote_ref(repo: Repo, name: str, local_ref: str, remote_ref: str) -> None: - """ - Shallow fetch remote reference so it is available locally - """ - - remote = repo.remote(name) - - # Initially fetch revision at depth 1 - LOGGER.info("Fetching remote ref %s from remote %s at depth 1", remote_ref, remote) - fetch_info = remote.fetch(remote_ref, depth=1, verbose=True, progress=GitProgressPrinter()) - - # If last commit for revision is in the fetch window, expand depth - # This check is necessary because some servers will throw an error when there are - # no commits in the fetch window - if fetch_info[-1].commit.committed_date >= approxidate.approx(config.since): - LOGGER.info( - 'Fetching ref %s from remote %s shallow since "%s"', - remote_ref, - remote, - config.since, - ) - try: - remote.fetch( - remote_ref, - shallow_since=config.since, - verbose=True, - progress=GitProgressPrinter(), - ) - except git.GitCommandError as e: - # ADO repos do not currently support --shallow-since, only depth - if "Server does not support --shallow-since" in e.stderr: - LOGGER.warning( - "Server does not support --shallow-since, retrying fetch without option." - ) - remote.fetch(remote_ref, verbose=True, progress=GitProgressPrinter()) - else: - raise - else: - LOGGER.info( - 'Newest commit for ref %s from remote %s is older than fetch window "%s"', - remote_ref, - remote, - config.since, - ) - - # Create tag at FETCH_HEAD to preserve reference locally - if not hasattr(repo.references, local_ref): - repo.create_tag(local_ref, "FETCH_HEAD") - - def monitor_downstream(): """ Cycle through downstream remotes and search for missing commits @@ -289,7 +200,7 @@ def monitor_downstream(): remote_ref, subject.distroID, ) - fetch_remote_ref(repo, subject.distroID, local_ref, remote_ref) + repo.fetch_remote_ref(subject.distroID, local_ref, remote_ref) LOGGER.info( "(%d of %d) Monitoring Script starting for distro: %s, revision: %s", diff --git a/comma/util/symbols.py b/comma/util/symbols.py index 9ec95ed..f72cb8c 100755 --- a/comma/util/symbols.py +++ b/comma/util/symbols.py @@ -55,8 +55,7 @@ def map_symbols_to_patch( initial_reference = repo.head.reference try: - repo.head.reference = repo.commit(prev_commit) - repo.head.reset(index=True, working_tree=True) + repo.checkout(prev_commit) before_patch_apply = None # Iterate through commits @@ -66,8 +65,7 @@ def map_symbols_to_patch( before_patch_apply = get_symbols(repo.working_tree_dir, files) # Checkout commit - repo.head.reference = repo.commit(commit) - repo.head.reset(index=True, working_tree=True) + repo.checkout(commit) # Get symbols after patch is applied after_patch_apply = get_symbols(repo.working_tree_dir, files) @@ -87,8 +85,7 @@ def map_symbols_to_patch( finally: # Reset reference - repo.head.reference = initial_reference - repo.head.reset(index=True, working_tree=True) + repo.checkout(initial_reference) def get_hyperv_patch_symbols(): diff --git a/comma/util/tracking.py b/comma/util/tracking.py index ed5f287..ef16ebc 100644 --- a/comma/util/tracking.py +++ b/comma/util/tracking.py @@ -9,6 +9,7 @@ import pathlib import re from typing import Any, Iterable, List, Optional, Set, Tuple +import approxidate import git from comma.util import config @@ -115,6 +116,106 @@ class Repo: return self._tracked_paths + def fetch_remote_ref(self, remote: str, local_ref: str, remote_ref: str) -> None: + """ + Shallow fetch remote reference so it is available locally + """ + + remote = self.obj.remote(remote) + + # Initially fetch revision at depth 1 + LOGGER.info("Fetching remote ref %s from remote %s at depth 1", remote_ref, remote) + fetch_info = remote.fetch(remote_ref, depth=1, verbose=True, progress=GitProgressPrinter()) + + # If last commit for revision is in the fetch window, expand depth + # This check is necessary because some servers will throw an error when there are + # no commits in the fetch window + if fetch_info[-1].commit.committed_date >= approxidate.approx(config.since): + LOGGER.info( + 'Fetching ref %s from remote %s shallow since "%s"', + remote_ref, + remote, + config.since, + ) + try: + remote.fetch( + remote_ref, + shallow_since=config.since, + verbose=True, + progress=GitProgressPrinter(), + ) + except git.GitCommandError as e: + # ADO repos do not currently support --shallow-since, only depth + if "Server does not support --shallow-since" in e.stderr: + LOGGER.warning( + "Server does not support --shallow-since, retrying fetch without option." + ) + remote.fetch(remote_ref, verbose=True, progress=GitProgressPrinter()) + else: + raise + else: + LOGGER.info( + 'Newest commit for ref %s from remote %s is older than fetch window "%s"', + remote_ref, + remote, + config.since, + ) + + # Create tag at FETCH_HEAD to preserve reference locally + if not hasattr(self.obj.references, local_ref): + self.obj.create_tag(local_ref, "FETCH_HEAD") + + def get_missing_cherries(self, reference, paths): + """ + Get a list of cherry-picked commits missing from the downstream reference + """ + + # Get all upstream commits on tracked paths within window + upstream_commits = set( + self.obj.git.log( + "--no-merges", + "--pretty=format:%H", + f"--since={config.since}", + "origin/master", + "--", + paths, + ).splitlines() + ) + + # Get missing cherries for all paths, but don't filter by path since it takes forever + missing_cherries = set( + self.obj.git.log( + "--no-merges", + "--right-only", + "--cherry-pick", + "--pretty=format:%H", + f"--since={config.since}", + f"{reference}...origin/master", + ).splitlines() + ) + + return missing_cherries & upstream_commits + + def get_remote_tags(self, remote: str): + """ + List tags for a given remote in the format tags/TAGNAME + """ + + return tuple( + line.split("/", 1)[-1] + for line in self.obj.git.ls_remote( + "--tags", "--refs", "--sort=v:refname", remote + ).splitlines() + ) + + def checkout(self, reference): + """ + Checkout the given reference + """ + + self.obj.head.reference = self.obj.commit(reference) + self.obj.head.reset(index=True, working_tree=True) + class Session: """