From 27442af2040e554a09785085463bfdcecb36ecb8 Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Thu, 29 Dec 2011 21:13:08 +0000 Subject: [PATCH] Add tool that downloads gm baseline images from the bots. Review URL: http://codereview.appspot.com/5503082 git-svn-id: http://skia.googlecode.com/svn/trunk@2937 2bbb7eff-a529-9590-31e7-b0007b416f81 --- tools/download-baselines.py | 141 ++++++++++++++++++++++++++++++++++++ tools/svn.py | 66 +++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 tools/download-baselines.py create mode 100644 tools/svn.py diff --git a/tools/download-baselines.py b/tools/download-baselines.py new file mode 100644 index 000000000..fa0e0b302 --- /dev/null +++ b/tools/download-baselines.py @@ -0,0 +1,141 @@ +''' +Downloads the actual gm results most recently generated by the Skia buildbots, +and adds any new ones to SVN control. + +This tool makes it much easier to check in new baselines, via the following +steps: + +cd .../trunk +svn update +# make sure there are no files awaiting svn commit +tools/download-baselines.py gm/base-macmini-lion-fixed # or other gm/ subdir +# upload CL for review +# validate that the new images look right +# commit CL + + +Copyright 2011 Google Inc. + +Use of this source code is governed by a BSD-style license that can be +found in the LICENSE file. +''' + +# common Python modules +import re +import sys +import urllib2 + +# modules declared within this same directory +import svn + +# Where to download recently generated baseline images for each baseline type. +# +# For now this only works for our Mac buildbots; our other buildbots aren't +# uploading their results to a web server yet. +# +# Note also that these will currently work only within the Google corporate +# network; that will also change soon. +ACTUALS_BY_BASELINE_SUBDIR = { + 'gm/base-macmini': + 'http://172.29.92.185/b/build/slave/Skia_Mac_Float_NoDebug/gm/actual', + 'gm/base-macmini-fixed': + 'http://172.29.92.185/b/build/slave/Skia_Mac_Fixed_NoDebug/gm/actual', + 'gm/base-macmini-lion-fixed': + 'http://172.29.92.179/b/build/slave/Skia_MacMiniLion_Fixed_NoDebug/gm/actual', + 'gm/base-macmini-lion-float': + 'http://172.29.92.179/b/build/slave/Skia_MacMiniLion_Float_NoDebug/gm/actual', +} + +IMAGE_REGEX = '.+\.png' +IMAGE_MIMETYPE = 'image/png' + +def GetPlatformUrl(baseline_subdir): + """Return URL within which the buildbots store generated baseline images, + as of multiple svn revisions. + + Raises KeyError if we don't have a URL matching this baseline_subdir. + + @param baseline_subdir indicates which platform we want images for + """ + try: + return ACTUALS_BY_BASELINE_SUBDIR[baseline_subdir] + except KeyError: + raise KeyError( + 'unknown baseline_subdir "%s", try one of these instead: %s' % ( + baseline_subdir, ACTUALS_BY_BASELINE_SUBDIR.keys())) + +def GetLatestResultsUrl(baseline_subdir): + """Return URL from which we can download the MOST RECENTLY generated + images for this baseline type. + + @param baseline_subdir indicates which platform we want images for + """ + base_platform_url = GetPlatformUrl(baseline_subdir) + print 'base_platform_url is %s' % base_platform_url + + # Find the most recently generated baseline images within base_platform_url + response = urllib2.urlopen(base_platform_url) + html = response.read() + link_regex = re.compile('') + links = link_regex.findall(html) + last_link = links[-1] + most_recent_result_url = '%s/%s' % (base_platform_url, last_link) + print 'most_recent_result_url is %s' % most_recent_result_url + return most_recent_result_url + +def DownloadMatchingFiles(source_url, filename_regex, dest_dir): + """Download all files from source_url that match filename_regex, and save + them (with their original filenames) in dest_dir. + + @param source_url + @param filename_regex + @param dest_dir + """ + while source_url.endswith('/'): + source_url = source_url[:-1] + response = urllib2.urlopen(source_url) + html = response.read() + link_regex = re.compile('' % filename_regex) + links = link_regex.findall(html) + for link in links: + DownloadBinaryFile('%s/%s' % (source_url, link), + '%s/%s' % (dest_dir, link)) + +def DownloadBinaryFile(source_url, dest_path): + """Download a single file from its source_url and save it to local disk + at dest_path. + + @param source_url + @param dest_path + """ + print 'DownloadBinaryFile: %s -> %s' % (source_url, dest_path) + url_fh = urllib2.urlopen(source_url) + local_fh = open(dest_path, 'wb') + local_fh.write(url_fh.read()) + local_fh.close() + +def Main(arglist): + """Download most recently generated baseline images for a given platform, + and add any new ones to SVN control. + + @param arglist sys.argv or equivalent + """ + num_args = len(arglist) + if num_args != 2: + raise Exception('usage: %s ' % __file__) + + baseline_subdir = arglist[1] + while baseline_subdir.endswith('/'): + baseline_subdir = baseline_subdir[:-1] + + results_url = GetLatestResultsUrl(baseline_subdir) + DownloadMatchingFiles(results_url, IMAGE_REGEX, baseline_subdir) + svn_handler = svn.Svn(baseline_subdir) + new_files = svn_handler.GetNewFiles() + if new_files: + svn_handler.AddFiles(new_files) + svn_handler.SetProperty(new_files, svn.PROPERTY_MIMETYPE, + IMAGE_MIMETYPE) + +if __name__ == '__main__': + Main(sys.argv) diff --git a/tools/svn.py b/tools/svn.py new file mode 100644 index 000000000..e87830176 --- /dev/null +++ b/tools/svn.py @@ -0,0 +1,66 @@ +''' +Copyright 2011 Google Inc. + +Use of this source code is governed by a BSD-style license that can be +found in the LICENSE file. +''' + +import re +import subprocess + +PROPERTY_MIMETYPE = 'svn:mime-type' + +class Svn: + + def __init__(self, directory): + """Set up to manipulate SVN control within the given directory. + + @param directory + """ + self._directory = directory + + def _RunCommand(self, args): + """Run a command (from self._directory) and return stdout as a single + string. + + @param args a list of arguments + """ + proc = subprocess.Popen(args, cwd=self._directory, + stdout=subprocess.PIPE) + stdout = proc.communicate()[0] + returncode = proc.returncode + if returncode is not 0: + raise Exception('command "%s" failed in dir "%s": returncode=%s' % + (args, self._directory, returncode)) + return stdout + + def GetNewFiles(self): + """Return a list of files which are in this directory but NOT under + SVN control. + """ + stdout = self._RunCommand(['svn', 'status']) + new_regex = re.compile('\? +(.+)') + files = new_regex.findall(stdout) + return files + + def AddFiles(self, filenames): + """Adds these files to SVN control. + + @param filenames files to add to SVN control + """ + args = ['svn', 'add'] + args.extend(filenames) + print '\n\nAddFiles: %s' % args + print self._RunCommand(args) + + def SetProperty(self, filenames, property_name, property_value): + """Sets a svn property for these files. + + @param filenames files to set property on + @param property_name property_name to set for each file + @param property_value what to set the property_name to + """ + args = ['svn', 'propset', property_name, property_value] + args.extend(filenames) + print '\n\nSetProperty: %s' % args + print self._RunCommand(args)