From 93acfd2b494b502b90257b0924ffa220912e9ce5 Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Tue, 3 Sep 2013 18:27:19 +0000 Subject: [PATCH] rebaseline.py: fix --add-new when there are no expectations at all BUG=skia:1582 R=rmistry@google.com Review URL: https://codereview.chromium.org/23899003 git-svn-id: http://skia.googlecode.com/svn/trunk@11063 2bbb7eff-a529-9590-31e7-b0007b416f81 --- tools/rebaseline.py | 58 ++++++------- .../actual-results.json | 24 ++--- .../expected-results.json | 27 +----- .../add-new/output-expected/command_line | 1 + .../expected-results.json | 87 +++++++++++++++++++ .../expected-results.json | 36 ++++++++ .../expected-results.json | 85 ++++++++++++++++++ .../add-new/output-expected/return_value | 1 + .../output/add-new/output-expected/stdout | 0 .../expected-results.json | 27 +----- tools/tests/run.sh | 1 + 11 files changed, 250 insertions(+), 97 deletions(-) create mode 100644 tools/tests/rebaseline/output/add-new/output-expected/command_line create mode 100644 tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/expected-results.json create mode 100644 tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json create mode 100644 tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Win7-ShuttleA-HD2000-x86-Release/expected-results.json create mode 100644 tools/tests/rebaseline/output/add-new/output-expected/return_value create mode 100644 tools/tests/rebaseline/output/add-new/output-expected/stdout diff --git a/tools/rebaseline.py b/tools/rebaseline.py index bf21e87f3..a1e3c576b 100755 --- a/tools/rebaseline.py +++ b/tools/rebaseline.py @@ -94,47 +94,42 @@ TEST_BUILDERS = [ class _InternalException(Exception): pass -# Object that handles exceptions, either raising them immediately or collecting -# them to display later on. class ExceptionHandler(object): + """ Object that handles exceptions, either raising them immediately or + collecting them to display later on.""" # params: - # keep_going_on_failure: if False, report failures and quit right away; - # if True, collect failures until - # ReportAllFailures() is called def __init__(self, keep_going_on_failure=False): + """ + params: + keep_going_on_failure: if False, report failures and quit right away; + if True, collect failures until + ReportAllFailures() is called + """ self._keep_going_on_failure = keep_going_on_failure self._failures_encountered = [] - self._exiting = False - # Exit the program with the given status value. - def _Exit(self, status=1): - self._exiting = True - sys.exit(status) - - # We have encountered an exception; either collect the info and keep going, - # or exit the program right away. - def RaiseExceptionOrContinue(self, e): - # If we are already quitting the program, propagate any exceptions - # so that the proper exit status will be communicated to the shell. - if self._exiting: - raise e + def RaiseExceptionOrContinue(self): + """ We have encountered an exception; either collect the info and keep + going, or exit the program right away.""" + # Get traceback information about the most recently raised exception. + exc_info = sys.exc_info() if self._keep_going_on_failure: - print >> sys.stderr, 'WARNING: swallowing exception %s' % e - self._failures_encountered.append(e) + print >> sys.stderr, ('WARNING: swallowing exception %s' % + repr(exc_info[1])) + self._failures_encountered.append(exc_info) else: - print >> sys.stderr, e print >> sys.stderr, ( 'Halting at first exception; to keep going, re-run ' + 'with the --keep-going-on-failure option set.') - self._Exit() + raise exc_info[1], None, exc_info[2] def ReportAllFailures(self): if self._failures_encountered: print >> sys.stderr, ('Encountered %d failures (see above).' % len(self._failures_encountered)) - self._Exit() + sys.exit(1) # Object that rebaselines a JSON expectations file (not individual image files). @@ -272,7 +267,10 @@ class JsonRebaseliner(object): expectations_input_filepath = os.path.join( self._expectations_root, builder, self._expectations_input_filename) expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) - expected_results = expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] + expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) + if not expected_results: + expected_results = {} + expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results # Update the expectations in memory, skipping any tests/configs that # the caller asked to exclude. @@ -330,7 +328,6 @@ parser.add_argument('--actuals-filename', 'of ACTUALS_BASE_URL) to read a summary of results ' 'from; defaults to %(default)s'), default='actual-results.json') -# TODO(epoger): Add test that exercises --add-new argument. parser.add_argument('--add-new', action='store_true', help=('in addition to the standard behavior of ' 'updating expectations for failing tests, add ' @@ -417,10 +414,13 @@ for builder in builders: mark_unreviewed=args.unreviewed) try: rebaseliner.RebaselineSubdir(builder=builder) - except BaseException as e: - exception_handler.RaiseExceptionOrContinue(e) + except: + exception_handler.RaiseExceptionOrContinue() else: - exception_handler.RaiseExceptionOrContinue(_InternalException( - 'expectations_json_file %s not found' % expectations_json_file)) + try: + raise _InternalException('expectations_json_file %s not found' % + expectations_json_file) + except: + exception_handler.RaiseExceptionOrContinue() exception_handler.ReportAllFailures() diff --git a/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/actual-results.json b/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/actual-results.json index fa1aaf766..d0afc3fba 100644 --- a/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/actual-results.json +++ b/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/actual-results.json @@ -1,34 +1,26 @@ { "actual-results" : { - "failed" : { - "aaclip_565.png" : [ "bitmap-64bitMD5", 6190901827590820995 ], - "aaclip_8888.png" : [ "bitmap-64bitMD5", 14456211900777561488 ] - }, + "failed" : null, "failure-ignored" : null, "no-comparison" : { + "aaclip_565.png" : [ "bitmap-64bitMD5", 6190901827590820995 ], + "aaclip_8888.png" : [ "bitmap-64bitMD5", 14456211900777561488 ], + "aaclip_gpu.png" : [ "bitmap-64bitMD5", 11899819492385205974 ], "aaclip_pdf.png" : [ "bitmap-64bitMD5", 13515207506164874564 ] }, - "succeeded" : { - "aaclip_gpu.png" : [ "bitmap-64bitMD5", 11899819492385205974 ] - } + "succeeded" : null }, "expected-results" : { "aaclip_565.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 12345 ] - ], + "allowed-digests" : null, "ignore-failure" : false }, "aaclip_8888.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 67890 ] - ], + "allowed-digests" : null, "ignore-failure" : false }, "aaclip_gpu.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 11899819492385205974 ] - ], + "allowed-digests" : null, "ignore-failure" : false }, "aaclip_pdf.png" : { diff --git a/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json b/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json index 79af0cf5e..0967ef424 100644 --- a/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json +++ b/tools/tests/rebaseline/input/json1/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json @@ -1,26 +1 @@ -{ - "expected-results" : { - "aaclip_565.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 12345 ] - ], - "ignore-failure" : false - }, - "aaclip_8888.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 67890 ] - ], - "ignore-failure" : false - }, - "aaclip_gpu.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 11899819492385205974 ] - ], - "ignore-failure" : false - }, - "aaclip_pdf.png" : { - "allowed-digests" : null, - "ignore-failure" : false - } - } -} +{} diff --git a/tools/tests/rebaseline/output/add-new/output-expected/command_line b/tools/tests/rebaseline/output/add-new/output-expected/command_line new file mode 100644 index 000000000..4974601b6 --- /dev/null +++ b/tools/tests/rebaseline/output/add-new/output-expected/command_line @@ -0,0 +1 @@ +python tools/rebaseline.py --expectations-root tools/tests/rebaseline/output/add-new/output-actual/gm-expectations --actuals-base-url tools/tests/rebaseline/input/json1 --add-new --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release Test-Win7-ShuttleA-HD2000-x86-Release diff --git a/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/expected-results.json b/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/expected-results.json new file mode 100644 index 000000000..65f8e3206 --- /dev/null +++ b/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/expected-results.json @@ -0,0 +1,87 @@ +{ + "expected-results": { + "3x3bitmaprect_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 16998423976396106083 + ] + ], + "ignore-failure": false + }, + "3x3bitmaprect_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 2054956815327187963 + ] + ], + "ignore-failure": false + }, + "aaclip_gpu.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 11899819492385205974 + ] + ], + "ignore-failure": false, + "unknown-extra-field": "make sure that rebaseline.py maintains this unknown field within a record whose allowed-digest IS NOT modified" + }, + "aarectmodes_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 14760033689012826769 + ] + ], + "ignore-failure": false + }, + "imageblur_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3359963596899141322 + ] + ], + "ignore-failure": false, + "unknown-extra-field": "make sure that rebaseline.py maintains this unknown field within a record whose allowed-digest IS modified" + }, + "imageblur_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 4217923806027861152 + ] + ], + "ignore-failure": false + }, + "shadertext3_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3713708307125704716 + ] + ], + "ignore-failure": false + }, + "xfermodeimagefilter_pdf.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 16502178848783208088 + ] + ], + "ignore-failure": false + }, + "xfermodes_pdf.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 9151974350149210736 + ] + ], + "ignore-failure": false + } + } +} \ No newline at end of file diff --git a/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json b/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json new file mode 100644 index 000000000..3c15dc873 --- /dev/null +++ b/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json @@ -0,0 +1,36 @@ +{ + "expected-results": { + "aaclip_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 6190901827590820995 + ] + ] + }, + "aaclip_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 14456211900777561488 + ] + ] + }, + "aaclip_gpu.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 11899819492385205974 + ] + ] + }, + "aaclip_pdf.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 13515207506164874564 + ] + ] + } + } +} \ No newline at end of file diff --git a/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Win7-ShuttleA-HD2000-x86-Release/expected-results.json b/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Win7-ShuttleA-HD2000-x86-Release/expected-results.json new file mode 100644 index 000000000..17ea319fc --- /dev/null +++ b/tools/tests/rebaseline/output/add-new/output-expected/gm-expectations/Test-Win7-ShuttleA-HD2000-x86-Release/expected-results.json @@ -0,0 +1,85 @@ +{ + "expected-results": { + "3x3bitmaprect_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 16998423976396106083 + ] + ], + "ignore-failure": false + }, + "3x3bitmaprect_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 2054956815327187963 + ] + ], + "ignore-failure": false + }, + "aaclip_gpu.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 11899819492385205974 + ] + ], + "ignore-failure": false + }, + "aarectmodes_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 14760033689012826769 + ] + ], + "ignore-failure": false + }, + "imageblur_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3359963596899141322 + ] + ], + "ignore-failure": false + }, + "imageblur_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 4217923806027861152 + ] + ], + "ignore-failure": false + }, + "shadertext3_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3713708307125704716 + ] + ], + "ignore-failure": false + }, + "xfermodeimagefilter_pdf.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 16502178848783208088 + ] + ], + "ignore-failure": false + }, + "xfermodes_pdf.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 9151974350149210736 + ] + ], + "ignore-failure": false + } + } +} \ No newline at end of file diff --git a/tools/tests/rebaseline/output/add-new/output-expected/return_value b/tools/tests/rebaseline/output/add-new/output-expected/return_value new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tools/tests/rebaseline/output/add-new/output-expected/return_value @@ -0,0 +1 @@ +0 diff --git a/tools/tests/rebaseline/output/add-new/output-expected/stdout b/tools/tests/rebaseline/output/add-new/output-expected/stdout new file mode 100644 index 000000000..e69de29bb diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json index 79af0cf5e..0967ef424 100644 --- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/expected-results.json @@ -1,26 +1 @@ -{ - "expected-results" : { - "aaclip_565.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 12345 ] - ], - "ignore-failure" : false - }, - "aaclip_8888.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 67890 ] - ], - "ignore-failure" : false - }, - "aaclip_gpu.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 11899819492385205974 ] - ], - "ignore-failure" : false - }, - "aaclip_pdf.png" : { - "allowed-digests" : null, - "ignore-failure" : false - } - } -} +{} diff --git a/tools/tests/run.sh b/tools/tests/run.sh index 6a41a8b3b..a6e418da6 100755 --- a/tools/tests/run.sh +++ b/tools/tests/run.sh @@ -244,6 +244,7 @@ REBASELINE_INPUT=tools/tests/rebaseline/input REBASELINE_OUTPUT=tools/tests/rebaseline/output rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Win7-ShuttleA-HD2000-x86-Release" "$REBASELINE_OUTPUT/using-json1-expectations" rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --bugs 1234 5678 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Win7-ShuttleA-HD2000-x86-Release --notes notes_content --unreviewed" "$REBASELINE_OUTPUT/marked-unreviewed" +rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --add-new --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release Test-Win7-ShuttleA-HD2000-x86-Release" "$REBASELINE_OUTPUT/add-new" # # Test jsondiff.py ...