Add tool that downloads gm baseline images from the bots.

Review URL: http://codereview.appspot.com/5503082

git-svn-id: http://skia.googlecode.com/svn/trunk@2937 2bbb7eff-a529-9590-31e7-b0007b416f81

tools/download-baselines.py

@@ -0,0 +1,141 @@
+'''
+Downloads the actual gm results most recently generated by the Skia buildbots,
+and adds any new ones to SVN control.
+
+This tool makes it much easier to check in new baselines, via the following
+steps:
+
+cd .../trunk
+svn update
+# make sure there are no files awaiting svn commit
+tools/download-baselines.py gm/base-macmini-lion-fixed  # or other gm/ subdir
+# upload CL for review
+# validate that the new images look right
+# commit CL
+
+
+Copyright 2011 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+# common Python modules
+import re
+import sys
+import urllib2
+
+# modules declared within this same directory
+import svn
+
+# Where to download recently generated baseline images for each baseline type.
+#
+# For now this only works for our Mac buildbots; our other buildbots aren't
+# uploading their results to a web server yet.
+#
+# Note also that these will currently work only within the Google corporate
+# network; that will also change soon.
+ACTUALS_BY_BASELINE_SUBDIR = {
+    'gm/base-macmini':
+        'http://172.29.92.185/b/build/slave/Skia_Mac_Float_NoDebug/gm/actual',
+    'gm/base-macmini-fixed':
+        'http://172.29.92.185/b/build/slave/Skia_Mac_Fixed_NoDebug/gm/actual',
+    'gm/base-macmini-lion-fixed':
+        'http://172.29.92.179/b/build/slave/Skia_MacMiniLion_Fixed_NoDebug/gm/actual',
+    'gm/base-macmini-lion-float':
+        'http://172.29.92.179/b/build/slave/Skia_MacMiniLion_Float_NoDebug/gm/actual',
+}
+
+IMAGE_REGEX = '.+\.png'
+IMAGE_MIMETYPE = 'image/png'
+
+def GetPlatformUrl(baseline_subdir):
+    """Return URL within which the buildbots store generated baseline images,
+    as of multiple svn revisions.
+
+    Raises KeyError if we don't have a URL matching this baseline_subdir.
+
+    @param baseline_subdir indicates which platform we want images for
+    """
+    try:
+        return ACTUALS_BY_BASELINE_SUBDIR[baseline_subdir]
+    except KeyError:
+        raise KeyError(
+            'unknown baseline_subdir "%s", try one of these instead: %s' % (
+                baseline_subdir, ACTUALS_BY_BASELINE_SUBDIR.keys()))
+
+def GetLatestResultsUrl(baseline_subdir):
+    """Return URL from which we can download the MOST RECENTLY generated
+    images for this baseline type.
+
+    @param baseline_subdir indicates which platform we want images for
+    """
+    base_platform_url = GetPlatformUrl(baseline_subdir)
+    print 'base_platform_url is %s' % base_platform_url
+
+    # Find the most recently generated baseline images within base_platform_url
+    response = urllib2.urlopen(base_platform_url)
+    html = response.read()
+    link_regex = re.compile('<a href="(.*)">')
+    links = link_regex.findall(html)
+    last_link = links[-1]
+    most_recent_result_url = '%s/%s' % (base_platform_url, last_link)
+    print 'most_recent_result_url is %s' % most_recent_result_url
+    return most_recent_result_url
+
+def DownloadMatchingFiles(source_url, filename_regex, dest_dir):
+    """Download all files from source_url that match filename_regex, and save
+    them (with their original filenames) in dest_dir.
+
+    @param source_url
+    @param filename_regex
+    @param dest_dir
+    """
+    while source_url.endswith('/'):
+        source_url = source_url[:-1]
+    response = urllib2.urlopen(source_url)
+    html = response.read()
+    link_regex = re.compile('<a href="(%s)">' % filename_regex)
+    links = link_regex.findall(html)
+    for link in links:
+        DownloadBinaryFile('%s/%s' % (source_url, link),
+                           '%s/%s' % (dest_dir, link))
+
+def DownloadBinaryFile(source_url, dest_path):
+    """Download a single file from its source_url and save it to local disk
+    at dest_path.
+
+    @param source_url
+    @param dest_path
+    """
+    print 'DownloadBinaryFile: %s -> %s' % (source_url, dest_path)
+    url_fh = urllib2.urlopen(source_url)
+    local_fh = open(dest_path, 'wb')
+    local_fh.write(url_fh.read())
+    local_fh.close()
+
+def Main(arglist):
+    """Download most recently generated baseline images for a given platform,
+    and add any new ones to SVN control.
+
+    @param arglist sys.argv or equivalent
+    """
+    num_args = len(arglist)
+    if num_args != 2:
+        raise Exception('usage: %s <baseline_subdir>' % __file__)
+
+    baseline_subdir = arglist[1]
+    while baseline_subdir.endswith('/'):
+        baseline_subdir = baseline_subdir[:-1]
+
+    results_url = GetLatestResultsUrl(baseline_subdir)
+    DownloadMatchingFiles(results_url, IMAGE_REGEX, baseline_subdir)
+    svn_handler = svn.Svn(baseline_subdir)
+    new_files = svn_handler.GetNewFiles()
+    if new_files:
+        svn_handler.AddFiles(new_files)
+        svn_handler.SetProperty(new_files, svn.PROPERTY_MIMETYPE,
+                                IMAGE_MIMETYPE)
+
+if __name__ == '__main__':
+    Main(sys.argv)

tools/svn.py

@@ -0,0 +1,66 @@
+'''
+Copyright 2011 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+import re
+import subprocess
+
+PROPERTY_MIMETYPE = 'svn:mime-type'
+
+class Svn:
+
+    def __init__(self, directory):
+        """Set up to manipulate SVN control within the given directory.
+
+        @param directory
+        """
+        self._directory = directory
+
+    def _RunCommand(self, args):
+        """Run a command (from self._directory) and return stdout as a single
+        string.
+
+        @param args a list of arguments
+        """
+        proc = subprocess.Popen(args, cwd=self._directory,
+                                stdout=subprocess.PIPE)
+        stdout = proc.communicate()[0]
+        returncode = proc.returncode
+        if returncode is not 0:
+            raise Exception('command "%s" failed in dir "%s": returncode=%s' %
+                            (args, self._directory, returncode))
+        return stdout
+
+    def GetNewFiles(self):
+        """Return a list of files which are in this directory but NOT under
+        SVN control.
+        """
+        stdout = self._RunCommand(['svn', 'status'])
+        new_regex = re.compile('\? +(.+)')
+        files = new_regex.findall(stdout)
+        return files
+
+    def AddFiles(self, filenames):
+        """Adds these files to SVN control.
+
+        @param filenames files to add to SVN control
+        """
+        args = ['svn', 'add']
+        args.extend(filenames)
+        print '\n\nAddFiles: %s' % args
+        print self._RunCommand(args)
+
+    def SetProperty(self, filenames, property_name, property_value):
+        """Sets a svn property for these files.
+
+        @param filenames files to set property on
+        @param property_name property_name to set for each file
+        @param property_value what to set the property_name to
+        """
+        args = ['svn', 'propset', property_name, property_value]
+        args.extend(filenames)
+        print '\n\nSetProperty: %s' % args
+        print self._RunCommand(args)