Merge pull request #90 from mozilla/regex-12132013

updates to regexes to handle job_types better

treeherder/etl/buildapi.py

@@ -594,13 +594,7 @@ class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin):
         if not buildername:
             return
 
-        job_type_target = ""
-
-        for job_type in buildbot.JOB_TYPE_BUILDERNAME:
-            for regex in buildbot.JOB_TYPE_BUILDERNAME[job_type]:
-                if regex.search(buildername):
-                    job_type_target = job_type
-                    break
+        job_type_target = buildbot.extract_job_type(buildername, default=None)
 
         if not job_type_target:
             self._load_missed_buildername(
@@ -626,9 +620,9 @@ class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin):
             self.report_obj[key]['data'][buildername]['count'] += 1
         else:
             self.report_obj[key]['data'][buildername] = {
-                'first_seen':self.t_stamp,
-                'count':1,
-                'objects':[]
+                'first_seen': self.t_stamp,
+                'count': 1,
+                'objects': []
                 }
 
             if build:

treeherder/etl/buildbot.py

@@ -187,6 +187,7 @@ PLATFORMS_BUILDERNAME = [
         }
     },
 
+    #// ** B2G **
 
     {
         'regex': re.compile('b2g.*_emulator-jb', re.IGNORECASE),
@@ -239,11 +240,15 @@ JOB_TYPE_BUILDERNAME = {
     'build': [
         re.compile('.+build'),
         re.compile('.+_dep'),
-        re.compile('.+(?<!l10n) nightly$'),     # all 'nightly'-s are builds
-        re.compile('.+ xulrunner$'),            # nightly
-        re.compile('.+ code coverage$'),        # nightly
+        re.compile('.+(?<!l10n)[ _]nightly$'),
+        re.compile('.+ xulrunner$'),
+        re.compile('.+ code coverage$'),
+        re.compile('.*valgrind$'),
+    ],
+    'unittest': [
+        re.compile('jetpack.*(opt|debug)$'),
+        re.compile('.+(?<!leak) test .+'),
     ],
-    'unittest': [re.compile('.+(?<!leak) test .+')],
     'talos': [re.compile('.+ talos .+')],
     'repack': [re.compile('.+ l10n .+')],
 }
@@ -272,6 +277,8 @@ TEST_NAME_BUILDERNAME = [
         #// If we start doing debug ASan tests, please kill these special build types
     {"regex": re.compile('debug asan nightly'), "desc": "AddressSanitizer Debug Nightly"},
     {"regex": re.compile('asan nightly'), "desc": "AddressSanitizer Opt Nightly"},
+    {"regex": re.compile('-br-haz'), "desc": "Static Rooting Hazard Analysis, Full Browser"},
+    {"regex": re.compile('-sh-haz'), "desc": "Static Rooting Hazard Analysis, JS Shell"},
     {"regex": re.compile('xulrunner'), "desc": "XULRunner Nightly"},
     {"regex": re.compile('b2g.*_hamachi_eng_nightly'), "desc": "Hamachi Device Image Nightly (Engineering)"},
     {"regex": re.compile('b2g.*_inari_eng_nightly'), "desc": "Inari Device Image Nightly (Engineering)"},
@@ -305,8 +312,6 @@ TEST_NAME_BUILDERNAME = [
     {"regex": re.compile('spidermonkey.*-generational'), "desc": "SpiderMonkey GGC Shell Build"},
     {"regex": re.compile('spidermonkey.*-exactroot'), "desc": "SpiderMonkey Exact Rooting Shell Build"},
     {"regex": re.compile('spidermonkey.*-warnaserr'), "desc": "SpiderMonkey --enable-sm-fail-on-warnings Build"},
-    {"regex": re.compile('-br-haz'), "desc": "Static Rooting Hazard Analysis, Full Browser"},
-    {"regex": re.compile('-sh-haz'), "desc": "Static Rooting Hazard Analysis, JS Shell"},
         #// If we start doing debug ASan tests, please kill these special build types
     {"regex": re.compile('debug asan build'), "desc": "AddressSanitizer Debug Build"},
     {"regex": re.compile('asan build'), "desc": "AddressSanitizer Opt Build"},
@@ -558,13 +563,12 @@ def extract_build_type(source_string):
     return output
 
 
-def extract_job_type(source_string):
-    job_type = 'build'
+def extract_job_type(source_string, default="build"):
     for job_type in JOB_TYPE_BUILDERNAME:
         for regex in JOB_TYPE_BUILDERNAME[job_type]:
             if regex.search(source_string):
                 return job_type
-    return job_type
+    return default
 
 
 def extract_name_info(source_string):

treeherder/webapp/media/builds4hanalysis/blacklist.json

@@ -1,5 +1,63 @@
 [
     "fuzzer-linux",
     "fuzzer-macosx64-lion",
-    "fuzzer-win64"
-]
+    "fuzzer-win64",
+    "release-comm-esr24-final_verification",
+    "release-comm-esr24-push_to_mirrors",
+    "release-comm-esr24-ready_for_releasetest_testing",
+    "release-comm-esr24-start_uptake_monitoring",
+    "release-mozilla-beta-almost_ready_for_release",
+    "release-mozilla-beta-android_repack_1/10",
+    "release-mozilla-beta-android_repack_10/10",
+    "release-mozilla-beta-android_repack_2/10",
+    "release-mozilla-beta-android_repack_3/10",
+    "release-mozilla-beta-android_repack_4/10",
+    "release-mozilla-beta-android_repack_5/10",
+    "release-mozilla-beta-android_repack_6/10",
+    "release-mozilla-beta-android_repack_7/10",
+    "release-mozilla-beta-android_repack_8/10",
+    "release-mozilla-beta-android_repack_9/10",
+    "release-mozilla-beta-antivirus",
+    "release-mozilla-beta-fennec_source",
+    "release-mozilla-beta-final_verification",
+    "release-mozilla-beta-linux64_update_verify_3/6",
+    "release-mozilla-beta-linux_update_verify_3/6",
+    "release-mozilla-beta-linux_update_verify_6/6",
+    "release-mozilla-beta-macosx64_update_verify_1/6",
+    "release-mozilla-beta-macosx64_update_verify_2/6",
+    "release-mozilla-beta-macosx64_update_verify_3/6",
+    "release-mozilla-beta-macosx64_update_verify_4/6",
+    "release-mozilla-beta-macosx64_update_verify_5/6",
+    "release-mozilla-beta-macosx64_update_verify_6/6",
+    "release-mozilla-beta-push_to_mirrors",
+    "release-mozilla-beta-ready_for_releasetest_testing",
+    "release-mozilla-beta-start_uptake_monitoring",
+    "release-mozilla-beta-win32_update_verify_1/6",
+    "release-mozilla-beta-win32_update_verify_2/6",
+    "release-mozilla-beta-win32_update_verify_3/6",
+    "release-mozilla-beta-win32_update_verify_4/6",
+    "release-mozilla-beta-win32_update_verify_5/6",
+    "release-mozilla-beta-win32_update_verify_6/6",
+    "release-mozilla-esr24-postrelease",
+    "Firefox mozilla-aurora linux l10n dep",
+    "Firefox mozilla-aurora linux64 l10n dep",
+    "Firefox mozilla-aurora macosx64 l10n dep",
+    "Firefox mozilla-aurora win32 l10n dep",
+    "Firefox mozilla-central linux l10n dep",
+    "Firefox mozilla-central linux64 l10n dep",
+    "Firefox mozilla-central macosx64 l10n dep",
+    "Firefox mozilla-central win32 l10n dep",
+    "Thunderbird comm-aurora linux l10n dep",
+    "Thunderbird comm-aurora linux64 l10n dep",
+    "Thunderbird comm-aurora macosx64 l10n dep",
+    "Thunderbird comm-aurora win32 l10n dep",
+    "Thunderbird comm-central linux l10n dep",
+    "Thunderbird comm-central linux64 l10n dep",
+    "Thunderbird comm-central macosx64 l10n dep",
+    "Thunderbird comm-central win32 l10n dep",
+    "Firefox birch win32 l10n nightly",
+    "release-mozilla-beta-android-armv6_build",
+    "release-mozilla-beta-android-x86_build",
+    "release-mozilla-beta-android_build",
+    "Firefox mozilla-central win32 l10n nightly"
+]