Bug 1792224: Implement vendoring individual files r=jewilde

Differential Revision: https://phabricator.services.mozilla.com/D158051

python/mozbuild/mozbuild/vendor/host_base.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
+import urllib
 import tempfile
 import subprocess
 
@@ -12,6 +13,7 @@ import subprocess
 class BaseHost:
     def __init__(self, manifest):
         self.manifest = manifest
+        self.repo_url = urllib.parse.urlparse(self.manifest["vendoring"]["url"])
 
     def upstream_tag(self, revision):
         """Temporarily clone the repo to get the latest tag and timestamp"""
@@ -58,4 +60,7 @@ class BaseHost:
             return (latest_tag, latest_tag_timestamp)
 
     def upstream_snapshot(self, revision):
-        raise Exception("Should not be called")
+        raise Exception("Unimplemented for this subclass...")
+
+    def upstream_path_to_file(self, revision, filepath):
+        raise Exception("Unimplemented for this subclass...")

python/mozbuild/mozbuild/vendor/host_github.py

@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import urllib
 import requests
 
 from mozbuild.vendor.host_base import BaseHost
@@ -14,8 +13,7 @@ class GitHubHost(BaseHost):
     def upstream_commit(self, revision):
         """Query the github api for a git commit id and timestamp."""
         github_api = "https://api.github.com"
-        repo_url = urllib.parse.urlparse(self.manifest["vendoring"]["url"])
-        repo = repo_url.path[1:]
+        repo = self.repo_url.path[1:]
         req = requests.get("/".join([github_api, "repos", repo, "commits", revision]))
         req.raise_for_status()
         info = req.json()
@@ -25,3 +23,7 @@ class GitHubHost(BaseHost):
         return "/".join(
             [self.manifest["vendoring"]["url"], "archive", revision + ".tar.gz"]
         )
+
+    def upstream_path_to_file(self, revision, filepath):
+        repo = self.repo_url.path[1:]
+        return "/".join(["https://raw.githubusercontent.com", repo, revision, filepath])

python/mozbuild/mozbuild/vendor/host_gitlab.py

@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import urllib
 import requests
 
 from mozbuild.vendor.host_base import BaseHost
@@ -13,9 +12,10 @@ from mozbuild.vendor.host_base import BaseHost
 class GitLabHost(BaseHost):
     def upstream_commit(self, revision):
         """Query the gitlab api for a git commit id and timestamp."""
-        repo_url = urllib.parse.urlparse(self.manifest["vendoring"]["url"])
-        gitlab_api = repo_url.scheme + "://" + repo_url.netloc + "/api/v4/projects/"
-        gitlab_api += repo_url.path[1:].replace("/", "%2F")
+        gitlab_api = (
+            self.repo_url.scheme + "://" + self.repo_url.netloc + "/api/v4/projects/"
+        )
+        gitlab_api += self.repo_url.path[1:].replace("/", "%2F")
         gitlab_api += "/repository/commits"
         req = requests.get("/".join([gitlab_api, revision]))
         req.raise_for_status()

python/mozbuild/mozbuild/vendor/vendor_manifest.py

@@ -93,6 +93,7 @@ class VendorManifest(MozbuildObject):
 
         ref_type = self.manifest["vendoring"].get("tracking", "commit")
         flavor = self.manifest["vendoring"].get("flavor", "regular")
+        # Individiual files are special
 
         if revision == "tip":
             # This case allows us to force-update a tag-tracking library to master
@@ -112,7 +113,7 @@ class VendorManifest(MozbuildObject):
             # We're up to date, don't do anything
             self.logInfo({}, "Latest upstream matches in-tree.")
             return
-        elif check_for_update:
+        elif flavor != "individual-file" and check_for_update:
             # Only print the new revision to stdout
             print("%s %s" % (new_revision, timestamp))
             return
@@ -157,8 +158,30 @@ class VendorManifest(MozbuildObject):
         self.update_yaml(new_revision, timestamp)
 
     def process_individual(self, new_revision, timestamp, ignore_modified):
+        # This design is used because there is no github API to query
+        # for the last commit that modified a file; nor a way to get file
+        # blame.  So really all we can do is just download and replace the
+        # files and see if they changed...
+        for f in self.manifest["vendoring"]["individual-files"]:
+            url = self.source_host.upstream_path_to_file(new_revision, f["upstream"])
+            self.logInfo(
+                {"local_file": f["destination"], "url": url},
+                "Downloading {local_file} from {url}...",
+            )
 
-        # TBD
+            with mozfile.NamedTemporaryFile() as tmpfile:
+                try:
+                    req = requests.get(url, stream=True)
+                    for data in req.iter_content(4096):
+                        tmpfile.write(data)
+                    tmpfile.seek(0)
+
+                    destination = self.get_full_path(f["destination"])
+                    shutil.copy2(tmpfile.name, destination)
+                except Exception as e:
+                    raise (e)
+
+        self.spurious_check(new_revision, ignore_modified)
 
         self.update_yaml(new_revision, timestamp)