Get the latest builds from treeherder. archive doesn't contain all builds.
This commit is contained in:
Hannes Verschore
Родитель
0f4b034abb
Коммит
6e6f0cccc3
|
|
@ -29,6 +29,8 @@ class DownloadTools(object):
|
|||
url.startswith("http://inbound-archive.pub.build.mozilla.org") or
|
||||
url.startswith("https://inbound-archive.pub.build.mozilla.org")):
|
||||
return ArchiveMozillaDownloader(url)
|
||||
if url.startswith("https://queue.taskcluster.net/v1/task/"):
|
||||
return TreeherderDownloader(url)
|
||||
if url.startswith("http://commondatastorage.googleapis.com"):
|
||||
return GoogleAPISDownloader(url)
|
||||
if (url.startswith("http://builds.nightly.webkit.org") or
|
||||
|
|
@ -102,8 +104,58 @@ class Downloader(object):
|
|||
zip.extractall(self.folder)
|
||||
zip.close()
|
||||
|
||||
class ArchiveMozillaDownloader(Downloader):
|
||||
class TreeherderDownloader(Downloader):
|
||||
|
||||
def __init__(self, url):
|
||||
super(TreeherderDownloader, self).__init__(url)
|
||||
self.url = "/".join(url.split("/")[:-1])+"/"
|
||||
self.filename = url.split("/")[-1]
|
||||
|
||||
def getfilename(self):
|
||||
return self.filename
|
||||
|
||||
def retrieveInfo(self):
|
||||
infoname = self.getinfoname()
|
||||
|
||||
response = urllib2.urlopen(self.url + infoname)
|
||||
raw_info = json.loads(response.read())
|
||||
|
||||
info = {}
|
||||
info["revision"] = raw_info["moz_source_stamp"]
|
||||
info["engine_type"] = "firefox"
|
||||
info["shell"] = False
|
||||
info["binary"] = os.path.abspath(self.getbinary())
|
||||
info["folder"] = os.path.abspath(self.folder)
|
||||
|
||||
return info
|
||||
|
||||
def getinfoname(self):
|
||||
filename = self.getfilename()
|
||||
try:
|
||||
filename = os.path.splitext(filename)[0]
|
||||
response = urllib2.urlopen(self.url + filename + ".json")
|
||||
html = response.read()
|
||||
except:
|
||||
filename = os.path.splitext(filename)[0]
|
||||
response = urllib2.urlopen(self.url + filename + ".json")
|
||||
html = response.read()
|
||||
|
||||
return filename + ".json"
|
||||
|
||||
def getbinary(self):
|
||||
if os.path.exists(self.folder + "firefox/firefox.exe"):
|
||||
return self.folder + "firefox/firefox.exe"
|
||||
if os.path.exists(self.folder + "firefox/firefox"):
|
||||
return self.folder + "firefox/firefox"
|
||||
files = os.listdir(self.folder)
|
||||
assert len(files) == 1
|
||||
if files[0].endswith(".apk"):
|
||||
return self.folder + files[0]
|
||||
if files[0].endswith(".dmg"):
|
||||
return self.folder + files[0]
|
||||
assert False
|
||||
|
||||
class ArchiveMozillaDownloader(Downloader):
|
||||
|
||||
def getfilename(self):
|
||||
try:
|
||||
|
|
@ -288,11 +340,17 @@ if __name__ == "__main__":
|
|||
parser.add_option("-u", "--url", dest="url",
|
||||
help="Specify a specific url to download.", default=None)
|
||||
parser.add_option("--repo", dest="repo",
|
||||
help="Specify a repo to download. Currently supports: mozilla-inbound, mozilla-central, mozilla-aurora, mozilla-beta, webkit, chrome", default=None)
|
||||
help="Specify a repo to download. Currently supports: mozilla-inbound, mozilla-central, mozilla-aurora, mozilla-beta, mozilla-release, webkit, chrome", default=None)
|
||||
parser.add_option("-r", dest="cset",
|
||||
help="Specify the revision to download. Defaults to 'latest'. (Note: this is currently only supported when using a mozilla repo)", default='latest')
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
import sys
|
||||
if ((sys.version_info.major < 2) or
|
||||
(sys.version_info.major == 2 and sys.version_info.minor < 7) or
|
||||
(sys.version_info.major == 2 and sys.version_info.minor == 7 and sys.version_info.micro < 10)):
|
||||
exit("python version need to be >= 2.7.10")
|
||||
|
||||
if options.url:
|
||||
downloader = DownloadTools.forSpecificUrl(options.url)
|
||||
elif options.repo:
|
||||
|
|
|
|||
|
|
@ -89,6 +89,8 @@ class MozillaUrlCreator(UrlCreator):
|
|||
return "mozilla-aurora-"+platform
|
||||
if self.repo == "mozilla-beta":
|
||||
return "mozilla-beta-"+platform
|
||||
if self.repo == "mozilla-release":
|
||||
return "mozilla-release-"+platform
|
||||
raise Exception("Unknown repo: " + self.repo)
|
||||
|
||||
def _url(self):
|
||||
|
|
@ -111,27 +113,23 @@ class MozillaUrlCreator(UrlCreator):
|
|||
return "osx-10-7"
|
||||
|
||||
def latest(self):
|
||||
response = urllib2.urlopen(self.url+"?C=N;O=D")
|
||||
html = response.read()
|
||||
ids = list(set(re.findall("([0-9]{5,})/", html)))
|
||||
ids = sorted(ids, reverse=True)
|
||||
return [self.url + id for id in ids]
|
||||
|
||||
def _build_id(self, id):
|
||||
url = "https://treeherder.mozilla.org/api/project/"+self.repo+"/jobs/?count=2000&result_set_id="+str(id)+""
|
||||
url = "https://treeherder.mozilla.org/api/project/"+self.repo+"/resultset/?count=10"
|
||||
data = utils.fetch_json(url)
|
||||
builds = [i for i in data["results"] if i["build_system_type"] == "buildbot"] # Builds
|
||||
builds = [i for i in builds if i["job_type_symbol"] == "B" or i["job_type_symbol"] == "Bo"] # Builds
|
||||
builds = [i for i in builds if i["platform"] == self.treeherder_platform()] # platform
|
||||
builds = [i for i in builds if i["platform_option"] == "opt"] # opt / debug / pgo
|
||||
|
||||
assert len(builds) == 1
|
||||
revisions = [i["revision"] for i in data["results"]]
|
||||
for revision in revisions:
|
||||
url = self._urlForRevision(revision)
|
||||
if len(url) == 1:
|
||||
return [url[0]]
|
||||
|
||||
url = "https://treeherder.mozilla.org/api/project/mozilla-inbound/job-log-url/?job_id="+str(builds[0]["id"])
|
||||
data = utils.fetch_json(url)
|
||||
return data[0]["url"].split("/")[-2]
|
||||
return []
|
||||
|
||||
def urlForRevision(self, cset):
|
||||
urls = self._urlForRevision(cset)
|
||||
assert len(urls) == 1
|
||||
return urls
|
||||
|
||||
def _urlForRevision(self, cset):
|
||||
# here we use a detour using treeherder to find the build_id,
|
||||
# corresponding to a revision.
|
||||
url = "https://treeherder.mozilla.org/api/project/"+self.repo+"/resultset/?full=false&revision="+cset
|
||||
|
|
@ -146,8 +144,41 @@ class MozillaUrlCreator(UrlCreator):
|
|||
if not data["results"][0]["revision"].startswith(cset):
|
||||
return None
|
||||
|
||||
build_id = self._build_id(data["results"][0]["id"])
|
||||
return [self._url()+str(build_id)+"/", self._archive_url()+str(build_id)+"/"]
|
||||
id = str(data["results"][0]["id"])
|
||||
|
||||
url = "https://treeherder.mozilla.org/api/project/"+self.repo+"/jobs/?count=2000&result_set_id="+str(id)+""
|
||||
data = utils.fetch_json(url)
|
||||
builds = data["results"]
|
||||
builds = [i for i in builds if i["build_system_type"] == "taskcluster"]
|
||||
builds = [i for i in builds if i["job_type_symbol"] == "B" or i["job_type_symbol"] == "Bo"] # Builds
|
||||
builds = [i for i in builds if i["platform"] == self.treeherder_platform()] # platform
|
||||
builds = [i for i in builds if i["platform_option"] == "opt"] # opt / debug / pgo
|
||||
|
||||
if len(builds) != 1:
|
||||
return []
|
||||
|
||||
url = "https://treeherder.mozilla.org/api/jobdetail/?job_guid="+str(builds[0]["job_guid"])
|
||||
data = utils.fetch_json(url)
|
||||
|
||||
urls = [item["url"] for item in data["results"] if item["url"]]
|
||||
urls = [i for i in urls if "info" not in i]
|
||||
urls = [i for i in urls if "testpackages" not in i]
|
||||
urls = [i for i in urls if "json" not in i]
|
||||
urls = [i for i in urls if "log" not in i]
|
||||
urls = [i for i in urls if "crash" not in i]
|
||||
urls = [i for i in urls if "test" not in i]
|
||||
urls = [i for i in urls if "mar" not in i]
|
||||
urls = [i for i in urls if "one-click-loaner" not in i]
|
||||
urls = [i for i in urls if "task-inspector" not in i]
|
||||
urls = [i for i in urls if "mbsdiff" not in i]
|
||||
urls = [i for i in urls if "checksums" not in i]
|
||||
urls = [i for i in urls if "xpi" not in i]
|
||||
urls = [i for i in urls if "txt" not in i]
|
||||
urls = [i for i in urls if "jsshell" not in i]
|
||||
urls = [i for i in urls if "mozharness" not in i]
|
||||
urls = [i for i in urls if "sdk" not in i]
|
||||
|
||||
return urls
|
||||
|
||||
def getUrlCreator(name):
|
||||
if "mozilla" in name:
|
||||
|
|
|
|||
|
|
@ -8,12 +8,13 @@ creators = [
|
|||
url_creator.getUrlCreator("mozilla-aurora"),
|
||||
url_creator.getUrlCreator("mozilla-beta"),
|
||||
url_creator.getUrlCreator("mozilla-central"),
|
||||
url_creator.getUrlCreator("mozilla-release"),
|
||||
url_creator.getUrlCreator("chrome"),
|
||||
url_creator.getUrlCreator("webkit")
|
||||
]
|
||||
|
||||
# Test 1
|
||||
for creator in creators:
|
||||
for creator in creators:
|
||||
urls = creator.find()
|
||||
assert urls
|
||||
assert len(urls) > 0
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче