Reland of Fail if an instrumentation test is missing size annotation

Changes since last one:
SiteSettingsPreferencesTest.testBackgroundSyncPermission now
named as a helper instead of annotated as a test.

TBR=jbudorick@chromium.org,sgurun@chromium.org,yfriedman@chromium.org,mathp@chromium.org
BUG=601464

Review-Url: https://codereview.chromium.org/1950793002
Cr-Original-Commit-Position: refs/heads/master@{#391421}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 01c2d6a9bdc61d422d8af9589ca0a0830d853ec1

android/pylib/instrumentation/instrumentation_test_instance.py

@@ -33,6 +33,8 @@ _DEFAULT_ANNOTATIONS = [
     'EnormousTest', 'IntegrationTest']
 _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [
     'DisabledTest', 'FlakyTest']
+_VALID_ANNOTATIONS = set(['Manual', 'PerfTest'] + _DEFAULT_ANNOTATIONS +
+                         _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS)
 _EXTRA_DRIVER_TEST_LIST = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList')
 _EXTRA_DRIVER_TEST_LIST_FILE = (
@@ -50,6 +52,13 @@ _NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE)
 _PICKLE_FORMAT_VERSION = 10
 
 
+class MissingSizeAnnotationError(Exception):
+  def __init__(self, class_name):
+    super(MissingSizeAnnotationError, self).__init__(class_name +
+        ': Test method is missing required size annotation. Add one of: ' +
+        ', '.join('@' + a for a in _VALID_ANNOTATIONS))
+
+
 # TODO(jbudorick): Make these private class methods of
 # InstrumentationTestInstance once the instrumentation test_runner is
 # deprecated.
@@ -579,6 +588,11 @@ class InstrumentationTestInstance(test_instance.TestInstance):
 
         all_annotations = dict(c['annotations'])
         all_annotations.update(m['annotations'])
+
+        # Enforce that all tests declare their size.
+        if not any(a in _VALID_ANNOTATIONS for a in all_annotations):
+          raise MissingSizeAnnotationError('%s.%s' % (c['class'], m['method']))
+
         if (not annotation_filter(all_annotations)
             or not excluded_annotation_filter(all_annotations)):
           continue