Add the perceptual difference metric to the rebaseline server

* Finds the location of the skpdiff binary. * Calculates the perceptual difference percentage using the skpdiff binary (treats -1 as 0% perceptual similarity). * Replaces the weightedDiffMeasure in view.html with perceptualDifferent. BUG=skia:2019 NOTRY=true Committed: http://code.google.com/p/skia/source/detail?r=13398 R=epoger@google.com Author: rmistry@google.com Review URL: https://codereview.chromium.org/147453003 git-svn-id: http://skia.googlecode.com/svn/trunk@13410 2bbb7eff-a529-9590-31e7-b0007b416f81
2014-02-11 18:21:26 +00:00 · 2014-02-11 18:21:26 +00:00 · 44546f85d9
--- a/gm/rebaseline_server/imagediffdb.py
+++ b/gm/rebaseline_server/imagediffdb.py
@ -10,10 +10,13 @@ Calulate differences between image pairs, and store them in a database.
 """

 import contextlib
+import csv
 import logging
 import os
 import re
 import shutil
+import sys
+import tempfile
 import urllib
 try:
  from PIL import Image, ImageChops
@ -21,6 +24,15 @@ except ImportError:
  raise ImportError('Requires PIL to be installed; see '
                    + 'http://www.pythonware.com/products/pil/')

+# Set the PYTHONPATH to include the tools directory.
+sys.path.append(
+    os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir,
+                        'tools'))
+import find_run_binary
+
+SKPDIFF_BINARY_NAME = 'skpdiff'
+
 DEFAULT_IMAGE_SUFFIX = '.png'
 DEFAULT_IMAGES_SUBDIR = 'images'

@ -99,6 +111,32 @@ class DiffRecord(object):
    whitediff_image = (graydiff_image.point(lambda p: p > 0 and VALUES_PER_BAND)
                                     .convert('1', dither=Image.NONE))

+    # Calculate the perceptual difference percentage.
+    skpdiff_csv_dir = tempfile.mkdtemp()
+    try:
+      skpdiff_csv_output = os.path.join(skpdiff_csv_dir, 'skpdiff-output.csv')
+      skpdiff_binary = find_run_binary.find_path_to_program(SKPDIFF_BINARY_NAME)
+      expected_img = os.path.join(storage_root, expected_images_subdir,
+                                  str(expected_image_locator) + image_suffix)
+      actual_img = os.path.join(storage_root, actual_images_subdir,
+                                str(actual_image_locator) + image_suffix)
+      find_run_binary.run_command(
+          [skpdiff_binary, '-p', expected_img, actual_img,
+           '--csv', skpdiff_csv_output, '-d', 'perceptual'])
+      with contextlib.closing(open(skpdiff_csv_output)) as csv_file:
+        for row in csv.DictReader(csv_file):
+          perceptual_similarity = float(row[' perceptual'].strip())
+          if not 0 <= perceptual_similarity <= 1:
+            # skpdiff outputs -1 if the images are different sizes. Treat any
+            # output that does not lie in [0, 1] as having 0% perceptual
+            # similarity.
+            perceptual_similarity = 0
+          # skpdiff returns the perceptual similarity, convert it to get the
+          # perceptual difference percentage.
+          self._perceptual_difference = 100 - (perceptual_similarity * 100)
+    finally:
+      shutil.rmtree(skpdiff_csv_dir)
+
    # Final touches on diff_image: use whitediff_image as an alpha mask.
    # Unchanged pixels are transparent; differing pixels are opaque.
    diff_image.putalpha(whitediff_image)
@ -128,6 +166,10 @@ class DiffRecord(object):
    return ((float(self._num_pixels_differing) * 100) /
            (self._width * self._height))

+  def get_perceptual_difference(self):
+    """Returns the perceptual difference percentage."""
+    return self._perceptual_difference
+
  def get_weighted_diff_measure(self):
    """Returns a weighted measure of image diffs, as a float between 0 and 100
    (inclusive)."""
--- a/gm/rebaseline_server/imagediffdb_test.py
+++ b/gm/rebaseline_server/imagediffdb_test.py
@ -19,7 +19,8 @@ import unittest
 import imagediffdb


-IMG_URL_BASE = 'http://chromium-skia-gm.commondatastorage.googleapis.com/gm/bitmap-64bitMD5/'
+IMG_URL_BASE = ('http://chromium-skia-gm.commondatastorage.googleapis.com/gm/'
+                'bitmap-64bitMD5/')


 class ImageDiffDbTest(unittest.TestCase):
@ -56,21 +57,22 @@ class ImageDiffDbTest(unittest.TestCase):
    # 3. actual image URL
    # 4. expected percent_pixels_differing (as a string, to 4 decimal places)
    # 5. expected weighted_diff_measure (as a string, to 4 decimal places)
-    # 6. expected max_diff_per_channel
+    # 6. expected perceptual difference (as a string, to 4 decimal places)
+    # 7. expected max_diff_per_channel
    selftests = [
        [
            'arcofzorro/16206093933823793653',
            IMG_URL_BASE + 'arcofzorro/16206093933823793653.png',
            'arcofzorro/13786535001616823825',
            IMG_URL_BASE + 'arcofzorro/13786535001616823825.png',
-            '0.0662', '0.0113', [255, 255, 247],
+            '0.0662', '0.0113', '0.0662', [255, 255, 247],
        ],
        [
            'gradients_degenerate_2pt/10552995703607727960',
            IMG_URL_BASE + 'gradients_degenerate_2pt/10552995703607727960.png',
            'gradients_degenerate_2pt/11198253335583713230',
            IMG_URL_BASE + 'gradients_degenerate_2pt/11198253335583713230.png',
-            '100.0000', '66.6667', [255, 0, 255],
+            '100.0000', '66.6667', '100.0000', [255, 0, 255],
        ],
    ]

@ -88,7 +90,8 @@ class ImageDiffDbTest(unittest.TestCase):
      self.assertEqual('%.4f' % record.get_percent_pixels_differing(),
                       selftest[4])
      self.assertEqual('%.4f' % record.get_weighted_diff_measure(), selftest[5])
-      self.assertEqual(record.get_max_diff_per_channel(), selftest[6])
+      self.assertEqual('%.4f' % record.get_perceptual_difference(), selftest[6])
+      self.assertEqual(record.get_max_diff_per_channel(), selftest[7])


 def main():
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@ -432,6 +432,7 @@ class Results(object):
            results_for_this_test['numDifferingPixels'] = 0
            results_for_this_test['percentDifferingPixels'] = 0
            results_for_this_test['weightedDiffMeasure'] = 0
+            results_for_this_test['perceptualDifference'] = 0
            results_for_this_test['maxDiffPerChannel'] = 0
          else:
            try:
@ -444,6 +445,8 @@ class Results(object):
                  diff_record.get_percent_pixels_differing())
              results_for_this_test['weightedDiffMeasure'] = (
                  diff_record.get_weighted_diff_measure())
+              results_for_this_test['perceptualDifference'] = (
+                  diff_record.get_perceptual_difference())
              results_for_this_test['maxDiffPerChannel'] = (
                  diff_record.get_max_diff_per_channel())
            except KeyError:
--- a/gm/rebaseline_server/static/view.html
+++ b/gm/rebaseline_server/static/view.html
@ -263,7 +263,7 @@
                   value="weightedDiffMeasure"
                   ng-checked="(sortColumn == 'weightedDiffMeasure')"
                   ng-click="sortResultsBy('weightedDiffMeasure')">
-            difference per pixel
+            perceptual difference
            <br>
            <input type="range" ng-model="pixelDiffBgColorBrightness"
                   ng-init="pixelDiffBgColorBrightness=64; pixelDiffBgColor=brightnessStringToHexColor(pixelDiffBgColorBrightness)"
@ -378,9 +378,9 @@
          <!-- diffs: per-channel RGB deltas -->
          <td valign="bottom" width="{{imageSize}}">
            <div ng-hide="result.expectedHashDigest == result.actualHashDigest"
-                 title="Weighted difference measure is {{result.weightedDiffMeasure.toFixed(4)}}%.  Maximum difference per channel: R={{result.maxDiffPerChannel[0]}}, G={{result.maxDiffPerChannel[1]}}, B={{result.maxDiffPerChannel[2]}}">
+                 title="Perceptual difference measure is {{result.perceptualDifference.toFixed(4)}}%.  Maximum difference per channel: R={{result.maxDiffPerChannel[0]}}, G={{result.maxDiffPerChannel[1]}}, B={{result.maxDiffPerChannel[2]}}">

-              {{result.weightedDiffMeasure.toFixed(4)}}%
+              {{result.perceptualDifference.toFixed(4)}}%
              {{result.maxDiffPerChannel}}
              <br/>
              <a href="/static/generated-images/diffs/{{result.expectedHashDigest}}-vs-{{result.actualHashDigest}}.png" target="_blank">View Image</a><br/>
--- a/gm/rebaseline_server/tests/outputs/expected/results_test.ResultsTest.test_gm/gm.json
+++ b/gm/rebaseline_server/tests/outputs/expected/results_test.ResultsTest.test_gm/gm.json
@ -53,6 +53,7 @@
      ], 
      "numDifferingPixels": 120000, 
      "percentDifferingPixels": 75.0, 
+      "perceptualDifference": 50.122499999999995, 
      "resultType": "failed", 
      "reviewed-by-human": null, 
      "test": "texdata", 
@ -76,6 +77,7 @@
      ], 
      "numDifferingPixels": 77691, 
      "percentDifferingPixels": 17.593070652173914, 
+      "perceptualDifference": 100, 
      "resultType": "failure-ignored", 
      "reviewed-by-human": false, 
      "test": "filterbitmap_checkerboard_192_192", 
@ -99,6 +101,7 @@
      ], 
      "numDifferingPixels": 82952, 
      "percentDifferingPixels": 18.784420289855074, 
+      "perceptualDifference": 100, 
      "resultType": "failure-ignored", 
      "reviewed-by-human": false, 
      "test": "filterbitmap_checkerboard_192_192", 
@ -122,6 +125,7 @@
      ], 
      "numDifferingPixels": 53150, 
      "percentDifferingPixels": 12.035778985507246, 
+      "perceptualDifference": 100, 
      "resultType": "failure-ignored", 
      "reviewed-by-human": false, 
      "test": "filterbitmap_checkerboard_32_2", 
@ -145,6 +149,7 @@
      ], 
      "numDifferingPixels": 53773, 
      "percentDifferingPixels": 12.17685688405797, 
+      "perceptualDifference": 100, 
      "resultType": "failure-ignored", 
      "reviewed-by-human": false, 
      "test": "filterbitmap_checkerboard_32_2", 
@ -208,6 +213,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "3x3bitmaprect", 
@ -227,6 +233,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "3x3bitmaprect", 
@ -244,6 +251,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": null, 
      "test": "aaclip", 
@ -261,6 +269,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": null, 
      "test": "aaclip", 
@ -278,6 +287,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": null, 
      "test": "displacement", 
@ -295,6 +305,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": null, 
      "test": "displacement", 
@ -312,6 +323,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": null, 
      "test": "displacement", 
@ -329,6 +341,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": null, 
      "test": "displacement", 
@ -348,6 +361,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "displacement", 
@ -422,6 +436,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "3x3bitmaprect", 
@ -441,6 +456,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "3x3bitmaprect", 
@ -460,6 +476,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "3x3bitmaprect", 
@ -479,6 +496,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "3x3bitmaprect", 
@ -498,6 +516,7 @@
      "maxDiffPerChannel": 0, 
      "numDifferingPixels": 0, 
      "percentDifferingPixels": 0, 
+      "perceptualDifference": 0, 
      "resultType": "succeeded", 
      "reviewed-by-human": false, 
      "test": "3x3bitmaprect",