servo: Merge #14400 - Add a log of only intermittents (from Manishearth:filter-intermittents); r=KiChjang

r? @KiChjang

Source-Repo: https://github.com/servo/servo
Source-Revision: 015c7642618e5be3df06a8799a87ec152f7d46fc

servo/etc/ci/buildbot_steps.yml

@@ -2,7 +2,7 @@ mac-rel-wpt1:
   - ./mach build --release
   - ./mach test-wpt-failure
   - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
-  - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
+  - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
   - ./mach build-cef --release
   - bash ./etc/ci/lockfile_changed.sh
@@ -11,7 +11,7 @@ mac-rel-wpt1:
 mac-rel-wpt2:
   - ./mach build --release
   - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
-  - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
+  - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
 
 mac-dev-unit:
   - ./mach build --dev
@@ -24,7 +24,7 @@ mac-dev-unit:
 mac-rel-css:
   - ./mach build --release
   - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
-  - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
+  - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
   - ./mach build-geckolib --release
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
@@ -61,13 +61,13 @@ linux-rel-wpt:
   - ./mach build --release --with-debug-assertions
   - ./mach test-wpt-failure
   - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
-  - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
+  - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
 
 linux-rel-css:
   - ./mach build --release --with-debug-assertions
   - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
-  - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
+  - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
   - ./mach build-cef --release --with-debug-assertions
   - ./mach build-geckolib --release
   - ./mach test-stylo --release

servo/python/servo/testing_commands.py

@@ -486,11 +486,13 @@ class MachCommands(CommandBase):
                      help="Error summary log to take un")
     @CommandArgument('--log-filteredsummary', default=None,
                      help='Print filtered log to file')
+    @CommandArgument('--log-intermittents', default=None,
+                     help='Print intermittents to file')
     @CommandArgument('--auth', default=None,
                      help='File containing basic authorization credentials for Github API (format `username:password`)')
     @CommandArgument('--use-tracker', default=False, action='store_true',
                      help='Use https://www.joshmatthews.net/intermittent-tracker')
-    def filter_intermittents(self, summary, log_filteredsummary, auth, use_tracker):
+    def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, auth, use_tracker):
         encoded_auth = None
         if auth:
             with open(auth, "r") as file:
@@ -502,6 +504,7 @@ class MachCommands(CommandBase):
                 if 'status' in line_json:
                     failures += [line_json]
         actual_failures = []
+        intermittents = []
         for failure in failures:
             if use_tracker:
                 query = urllib2.quote(failure['test'], safe='')
@@ -510,6 +513,8 @@ class MachCommands(CommandBase):
                 data = json.load(search)
                 if len(data) == 0:
                     actual_failures += [failure]
+                else:
+                    intermittents += [failure]
             else:
                 qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test']
                 # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+`
@@ -521,6 +526,14 @@ class MachCommands(CommandBase):
                 data = json.load(search)
                 if data['total_count'] == 0:
                     actual_failures += [failure]
+                else:
+                    intermittents += [failure]
+
+        if log_intermittents:
+            with open(log_intermittents, "w") as intermittents_file:
+                for intermittent in intermittents:
+                    json.dump(intermittent, intermittents_file)
+                    print("\n", end='', file=intermittents_file)
 
         if len(actual_failures) == 0:
             return 0