From c5b5430369ab2b8e2bafd3dc1a9074a16b0684cd Mon Sep 17 00:00:00 2001 From: Tom Ritter Date: Tue, 18 Oct 2022 16:56:07 +0000 Subject: [PATCH] Bug 1792224: Implement vendoring individual files r=jewilde Differential Revision: https://phabricator.services.mozilla.com/D158051 --- python/mozbuild/mozbuild/vendor/host_base.py | 7 ++++- .../mozbuild/mozbuild/vendor/host_github.py | 8 +++--- .../mozbuild/mozbuild/vendor/host_gitlab.py | 8 +++--- .../mozbuild/vendor/vendor_manifest.py | 27 +++++++++++++++++-- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/python/mozbuild/mozbuild/vendor/host_base.py b/python/mozbuild/mozbuild/vendor/host_base.py index 08bb753f9302..5c18e1ff1cd2 100644 --- a/python/mozbuild/mozbuild/vendor/host_base.py +++ b/python/mozbuild/mozbuild/vendor/host_base.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, print_function, unicode_literals import os +import urllib import tempfile import subprocess @@ -12,6 +13,7 @@ import subprocess class BaseHost: def __init__(self, manifest): self.manifest = manifest + self.repo_url = urllib.parse.urlparse(self.manifest["vendoring"]["url"]) def upstream_tag(self, revision): """Temporarily clone the repo to get the latest tag and timestamp""" @@ -58,4 +60,7 @@ class BaseHost: return (latest_tag, latest_tag_timestamp) def upstream_snapshot(self, revision): - raise Exception("Should not be called") + raise Exception("Unimplemented for this subclass...") + + def upstream_path_to_file(self, revision, filepath): + raise Exception("Unimplemented for this subclass...") diff --git a/python/mozbuild/mozbuild/vendor/host_github.py b/python/mozbuild/mozbuild/vendor/host_github.py index c15fe56f91a0..4a65f2c89814 100644 --- a/python/mozbuild/mozbuild/vendor/host_github.py +++ b/python/mozbuild/mozbuild/vendor/host_github.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, print_function, unicode_literals -import urllib import requests from mozbuild.vendor.host_base import BaseHost @@ -14,8 +13,7 @@ class GitHubHost(BaseHost): def upstream_commit(self, revision): """Query the github api for a git commit id and timestamp.""" github_api = "https://api.github.com" - repo_url = urllib.parse.urlparse(self.manifest["vendoring"]["url"]) - repo = repo_url.path[1:] + repo = self.repo_url.path[1:] req = requests.get("/".join([github_api, "repos", repo, "commits", revision])) req.raise_for_status() info = req.json() @@ -25,3 +23,7 @@ class GitHubHost(BaseHost): return "/".join( [self.manifest["vendoring"]["url"], "archive", revision + ".tar.gz"] ) + + def upstream_path_to_file(self, revision, filepath): + repo = self.repo_url.path[1:] + return "/".join(["https://raw.githubusercontent.com", repo, revision, filepath]) diff --git a/python/mozbuild/mozbuild/vendor/host_gitlab.py b/python/mozbuild/mozbuild/vendor/host_gitlab.py index 8f08675442e5..1cef1c409297 100644 --- a/python/mozbuild/mozbuild/vendor/host_gitlab.py +++ b/python/mozbuild/mozbuild/vendor/host_gitlab.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, print_function, unicode_literals -import urllib import requests from mozbuild.vendor.host_base import BaseHost @@ -13,9 +12,10 @@ from mozbuild.vendor.host_base import BaseHost class GitLabHost(BaseHost): def upstream_commit(self, revision): """Query the gitlab api for a git commit id and timestamp.""" - repo_url = urllib.parse.urlparse(self.manifest["vendoring"]["url"]) - gitlab_api = repo_url.scheme + "://" + repo_url.netloc + "/api/v4/projects/" - gitlab_api += repo_url.path[1:].replace("/", "%2F") + gitlab_api = ( + self.repo_url.scheme + "://" + self.repo_url.netloc + "/api/v4/projects/" + ) + gitlab_api += self.repo_url.path[1:].replace("/", "%2F") gitlab_api += "/repository/commits" req = requests.get("/".join([gitlab_api, revision])) req.raise_for_status() diff --git a/python/mozbuild/mozbuild/vendor/vendor_manifest.py b/python/mozbuild/mozbuild/vendor/vendor_manifest.py index b4735c2c8629..49ce45c776e4 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_manifest.py +++ b/python/mozbuild/mozbuild/vendor/vendor_manifest.py @@ -93,6 +93,7 @@ class VendorManifest(MozbuildObject): ref_type = self.manifest["vendoring"].get("tracking", "commit") flavor = self.manifest["vendoring"].get("flavor", "regular") + # Individiual files are special if revision == "tip": # This case allows us to force-update a tag-tracking library to master @@ -112,7 +113,7 @@ class VendorManifest(MozbuildObject): # We're up to date, don't do anything self.logInfo({}, "Latest upstream matches in-tree.") return - elif check_for_update: + elif flavor != "individual-file" and check_for_update: # Only print the new revision to stdout print("%s %s" % (new_revision, timestamp)) return @@ -157,8 +158,30 @@ class VendorManifest(MozbuildObject): self.update_yaml(new_revision, timestamp) def process_individual(self, new_revision, timestamp, ignore_modified): + # This design is used because there is no github API to query + # for the last commit that modified a file; nor a way to get file + # blame. So really all we can do is just download and replace the + # files and see if they changed... + for f in self.manifest["vendoring"]["individual-files"]: + url = self.source_host.upstream_path_to_file(new_revision, f["upstream"]) + self.logInfo( + {"local_file": f["destination"], "url": url}, + "Downloading {local_file} from {url}...", + ) - # TBD + with mozfile.NamedTemporaryFile() as tmpfile: + try: + req = requests.get(url, stream=True) + for data in req.iter_content(4096): + tmpfile.write(data) + tmpfile.seek(0) + + destination = self.get_full_path(f["destination"]) + shutil.copy2(tmpfile.name, destination) + except Exception as e: + raise (e) + + self.spurious_check(new_revision, ignore_modified) self.update_yaml(new_revision, timestamp)