Fix symbolication issues (#32)

This is a big set of changes.

It looks like the structure of crash pings have changed over the last
few years. This updates the code to handle the current version. Plus it
changes the code to work with the whole crash ping rather than pieces of
it. It specifies which pieces it needs in the comments.

This gets rid of the "windows flag" in favor of looking at the
normalized_os field of the crash ping.

This creates a SYMBOLICATION_API constant and moves it to the top level.

This fixes fx-crash-sig and sample.json so those work. This also updates
the examples in README.md.

This updates setup.py--this is a Python 3 library.

Finally, it fixes the issue where the output of symbolication puts the
debug_file (e.g. xul.pdb) in the module field of frames. Signature
generation expects that field to be the filename (e.g. xul.dll). I claim
that's a bug in the symbolication API code, but until it's fixed there,
we have to fix it in fx-crash-sig.

README.md

@@ -1,10 +1,10 @@
 # fx-crash-sig
 
-Get crash signature from Firefox crash trace
+Symbolicates crash pings and generates signatures.
 
-Take crash trace and:
+Take crash ping stack traces and:
 
-1. Use [tecken](https://github.com/mozilla-services/tecken) symbolication to symbolicate crash trace
+1. Use [tecken](https://github.com/mozilla-services/tecken) symbolication to symbolicate crash ping stack traces
 
 2. use [socorro-siggen](https://github.com/willkg/socorro-siggen) to get crash signature
 
@@ -20,13 +20,17 @@ pip install fx-crash-sig
 [Example script](/fx_crash_sig/example.py):
 
 ```py
-from fx_crash_sig import sample_traces
+import json
+
 from fx_crash_sig.crash_processor import CrashProcessor
 
+with open("crashping.json") as fp:
+    crash_ping = json.load(fp)
+
 crash_processor = CrashProcessor()
 
-signature = crash_processor.get_signature(sample_traces.trace1)
-    
+signature_result = crash_processor.get_signature(crash_ping)
+print(signature_result.signature)
 ```
 
 Command line (using [sample.json](/sample.json)):
@@ -34,3 +38,30 @@ Command line (using [sample.json](/sample.json)):
 ```sh
 cat sample.json | fx-crash-sig
 ```
+
+## Minimal crash ping structure
+
+These are the parts of the crash ping we use:
+
+```
+- normalized_os                (optional)
+- payload:
+  - metadata:
+    - async_shutdown_timeout   (optional)
+    - ipc_channel_error        (optional)
+    - oom_allocation_size      (optional)
+    - moz_crash_reason         (optional)
+  - stack_traces:
+    - crash_info:
+      - crashing_thread
+    - modules[]
+      - debug_file
+      - debug_id
+      - filename
+      - base_addr
+    - threads[]
+      - frames[]
+         - ip
+         - module_index
+         - trust
+```

fx_crash_sig/__init__.py

@@ -0,0 +1,5 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+SYMBOLICATION_API = "https://symbols.mozilla.org/symbolicate/v5"

fx_crash_sig/cmd_get_crash_sig.py

@@ -2,8 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import print_function
-
 import argparse
 import sys
 
@@ -11,6 +9,7 @@ import ujson as json
 
 from fx_crash_sig.crash_processor import CrashProcessor
 
+
 DESCRIPTION = """
 Takes raw crash trace and symbolicates it to return the crash signature
 """
@@ -18,23 +17,24 @@ Takes raw crash trace and symbolicates it to return the crash signature
 
 def cmdline():
     parser = argparse.ArgumentParser(description=DESCRIPTION)
-    parser.add_argument('-v', '--verbose', action='store_true')
-    parser.add_argument('-w', '--windows', action='store_true')
+    parser.add_argument("-v", "--verbose", action="store_true")
     args = parser.parse_args()
 
-    crash_processor = CrashProcessor(verbose=args.verbose,
-                                     windows=args.windows)
+    crash_processor = CrashProcessor(verbose=args.verbose)
     try:
         payload = json.loads(sys.stdin.read())
     except ValueError:
         if args.verbose:
-            print('fx-crash-sig: Failed: Invalid input format')
+            print("fx-crash-sig: Failed: Invalid input format")
         return
 
     try:
-        signature = crash_processor.get_signature(payload)
-        if signature is not None:
-            print(json.dumps(signature))
-    except Exception as e:
+        result = crash_processor.get_signature(payload)
+        if result is not None:
+            print(f"signature: {result.signature}")
+            print(f"notes:     ({len(result.notes)})")
+            for note in result.notes:
+                print(f"           * {note}")
+    except Exception as exc:
         if args.verbose:
-            print('fx-crash-sig: Failed: {}'.format(e.message))
+            print(f"fx-crash-sig: Failed: {exc!r}")

fx_crash_sig/crash_processor.py

@@ -2,65 +2,112 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import print_function
-
 import json
+
 from siggen.generator import SignatureGenerator
 
+from fx_crash_sig import SYMBOLICATION_API
 from fx_crash_sig.symbolicate import Symbolicator
 
 
 class CrashProcessor:
-    def __init__(self, max_frames=40,
-                 api_url='https://symbols.mozilla.org/symbolicate/v5',
-                 verbose=False,
-                 windows=False):
+    def __init__(self, max_frames=40, api_url=SYMBOLICATION_API, verbose=False):
         self.symbolicator = Symbolicator(max_frames, api_url, verbose)
         self.sig_generator = SignatureGenerator()
         self.verbose = verbose
-        self.windows = windows
 
-    def get_signature(self, payload):
-        symbolicated = self.symbolicate(payload)
-        signature = self.get_signature_from_symbolicated(symbolicated)
-        if self.verbose and len(signature['signature']) == 0:
-            print('fx-crash-sig: Failed siggen: {}'.format(signature['notes']))
-        return signature
+    def get_signature(self, crash_ping):
+        """Takes a crash_ping, symbolicates it, generates signature, returns result
 
-    def symbolicate(self, payload):
-        crash_data = payload.get('stackTraces', None)
-        if crash_data is None or len(crash_data) == 0:
+        :args dict crash_ping: a crash ping
+
+        :returns: signature result
+
+        """
+        symbolicated = self.symbolicate(crash_ping)
+        signature_result = self.get_signature_from_symbolicated(symbolicated)
+        if self.verbose and len(signature_result.signature) == 0:
+            print(f'fx-crash-sig: Failed siggen: {signature_result.notes}')
+        return signature_result
+
+    def symbolicate(self, crash_ping):
+        # These are the parts of the crash ping we use:
+        #
+        # - normalized_os
+        # - payload:
+        #   - crash_data
+        #   - metadata:
+        #     - async_shutdown_timeout
+        #     - ipc_channel_error
+        #     - oom_allocation_size
+        #     - moz_crash_reason
+        #   - stack_traces:
+        #     - crash_info:
+        #       - crashing_thread
+        #     - modules[]
+        #       - debug_file
+        #       - debug_id
+        #       - filename
+        #       - base_addr
+        #     - threads[]
+        #       - frames[]
+        #          - ip
+        #          - module_index
+        #          - trust
+        normalized_os = crash_ping.get("normalized_os") or ""
+        payload = crash_ping["payload"]
+
+        if isinstance(payload, str):
+            # If payload is a string, it's probably a JSON-encoded string
+            # straight from telemetry.crash. Try to decode it and if that
+            # fails, let the exception bubble up because there's nothing we can
+            # do with this crash report
+            payload = json.loads(payload)
+
+        metadata = payload.get("metadata", {})
+        stack_traces = payload["stack_traces"]
+
+        if stack_traces is None or len(stack_traces) == 0:
             symbolicated = {}
-        elif 'ipc_channel_error' in payload:
+        elif metadata.get("ipc_channel_error"):
             # ipc_channel_error will always overwrite the crash signature so
             # we don't need to symbolicate to get the signature
             symbolicated = {}
         else:
-            symbolicated = self.symbolicator.symbolicate(crash_data)
+            symbolicated = self.symbolicator.symbolicate(stack_traces)
 
-        metadata = payload['metadata']
+        metadata_fields = [
+            "async_shutdown_timeout",
+            "oom_allocation_size",
+            "moz_crash_reason",
+            "ipc_channel_error"
+        ]
 
-        meta_fields = {
-            'ipc_channel_error': 'ipc_channel_error',
-            'MozCrashReason': 'moz_crash_reason',
-            'OOMAllocationSize': 'oom_allocation_size',
-            'AsyncShutdownTimeout': 'async_shutdown_timeout'
-        }
+        symbolicated["os"] = (
+            "Windows NT" if normalized_os.startswith("Windows") else ""
+        )
+        for field_name in metadata_fields:
+            if metadata.get(field_name):
+                symbolicated[field_name] = metadata[field_name]
 
-        symbolicated['os'] = 'Windows NT' if self.windows else 'Not Windows'
-        for k in meta_fields.keys():
-            if k in metadata:
-                symbolicated[meta_fields[k]] = metadata[k]
-
-        # async_shutdown_timeout should be json string not dict
-        if 'async_shutdown_timeout' in metadata:
+        # async_shutdown_timeout should be json string not dict, so we need to
+        # encode it
+        if metadata.get("async_shutdown_timeout"):
             try:
-                metadata['async_shutdown_timeout'] = (
-                    json.dumps(metadata['async_shutdown_timeout']))
+                metadata["async_shutdown_timeout"] = json.dumps(
+                    metadata["async_shutdown_timeout"]
+                )
             except TypeError:
-                metadata.pop('async_shutdown_timeout')
+                metadata.pop("async_shutdown_timeout")
 
         return symbolicated
 
     def get_signature_from_symbolicated(self, symbolicated):
+        """Takes output of symbolicate() and returns a signature result
+
+        :args dict symbolicated: the result of .symbolicate()
+
+        :returns: a signature result
+
+        """
         return self.sig_generator.generate(symbolicated)

fx_crash_sig/example.py

@@ -2,8 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import print_function
-
 import ujson as json
 
 from fx_crash_sig import sample_traces

fx_crash_sig/symbolicate.py

@@ -2,16 +2,15 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import print_function
+from itertools import islice
 
 import requests
-from itertools import islice
+
+from fx_crash_sig import SYMBOLICATION_API
 
 
 class Symbolicator:
-    def __init__(self, max_frames=40,
-                 api_url='https://symbols.mozilla.org/symbolicate/v5',
-                 verbose=False):
+    def __init__(self, max_frames=40, api_url=SYMBOLICATION_API, verbose=False):
         self.max_frames = max_frames
         self.api_url = api_url
         self.empty_request = {'memoryMap': [], 'stacks': [], 'version': 5}
@@ -72,14 +71,16 @@ class Symbolicator:
                 ip_int = int(src_frame['ip'], 16)
                 out_frame['offset'] = src_frame['ip']
 
-                if 'module_index' not in src_frame:
+                if src_frame.get("module_index") is None:
+                    print(f"src_frame: {src_frame}")
                     continue
 
                 module_index = src_frame['module_index']
                 if not (module_index >= 0 and module_index < len(modules)):
-                    msg = "module_index " + module_index + " out of range for "
-                    msg += "thread " + thread_idx + " frame " + frame_idx
-                    raise ValueError(msg)
+                    raise ValueError(
+                        f"module {module_index} out of frange for thread {thread_idx} "
+                        f"frame {frame_idx}"
+                    )
 
                 module = modules[module_index]
 
@@ -140,15 +141,17 @@ class Symbolicator:
         except ValueError:
             return self.empty_request
 
-    def symbolicate(self, trace):
+    def symbolicate(self, stack_trace):
         """Symbolicate a single crash trace
 
-        :param dict trace: raw crash trace
+        :param dict stack_trace: raw crash trace from a crash_ping payload
+
         :return: dict: symbolicated trace
+
         """
-        if trace is None:
+        if stack_trace is None:
             return {}
-        symbolicated = self.symbolicate_multi([trace])
+        symbolicated = self.symbolicate_multi([stack_trace])
         return symbolicated if symbolicated is None else symbolicated[0]
 
     def symbolicate_multi(self, traces):
@@ -160,22 +163,35 @@ class Symbolicator:
         symbolication_requests = {
             'jobs': [self.__try_get_sym_req(t) for t in traces]
         }
-        crashing_threads = [t['crash_info'].get('crashing_thread', 0) for
-                            t in traces]
+        crashing_threads = [
+            t.get("crash_info", {}).get('crashing_thread', 0) if t else 0 for t in traces
+        ]
 
         try:
             symbolicated_list = self.__get_symbolicated_trace(symbolication_requests)
         except requests.HTTPError as e:
             if self.verbose:
-                print('fx-crash-sig: Failed Symbolication: {}'.format(e.message))
+                print(f'fx-crash-sig: Failed Symbolication: {e.message}')
             return None
 
+        debug_file_to_filename = {}
+        for trace in traces:
+            for module in trace["modules"]:
+                if "debug_file" in module and "filename" in module:
+                    debug_file_to_filename[module["debug_file"]] = module["filename"]
+
         # make into siggen suitable format
         formatted_symbolications = []
-        for result, crashing_thread in zip(symbolicated_list['results'],
-                                           crashing_threads):
+        for result, crashing_thread in zip(symbolicated_list['results'], crashing_threads):
             symbolicated = {'crashing_thread': crashing_thread, 'threads': []}
             for frames in result['stacks']:
+                # FIXME(willkg): Tecken symbolication API returns "module" as
+                # the debug_file (e.g. xul.pdb), but it should be the module
+                # filename (e.g.  xul.dll). We fix that here.
+                for frame in frames:
+                    if frame.get("module") and frame["module"] in debug_file_to_filename:
+                        frame["module"] = debug_file_to_filename[frame["module"]]
                 symbolicated['threads'].append({'frames': frames})
             formatted_symbolications.append(symbolicated)
+
         return formatted_symbolications

setup.py

@@ -13,20 +13,18 @@ def read_file(name):
 
 
 install_requires = [
-    'requests',
-    'siggen',
-    'ujson',
+    "requests",
+    "siggen>=1.0.0,<2.0.0",
+    "ujson",
 ]
 
 setup(
-    name='fx-crash-sig',
-    version='0.1.11',
-    description=' Get crash signature from Firefox crash trace ',
-    long_description=read_file('README.md'),
-    long_description_content_type='text/markdown',
-    maintainer='Ben Wu',
-    maintainer_email='bwub124@gmail.com',
-    url='https://github.com/Ben-Wu/fx-crash-sig',
+    name="fx-crash-sig",
+    version="0.1.11",
+    description="Get crash signature from Firefox crash ping",
+    long_description=read_file("README.md"),
+    long_description_content_type="text/markdown",
+    url="https://github.com/mozilla/fx-crash-sig",
     packages=find_packages(),
     include_package_data=True,
     install_requires=install_requires,
@@ -35,8 +33,13 @@ setup(
         fx-crash-sig=fx_crash_sig.cmd_get_crash_sig:cmdline
     """,
     classifiers=[
-        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
-        'Programming Language :: Python :: 2 :: Only',
-        'Development Status :: 2 - Pre-Alpha',
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Development Status :: 2 - Pre-Alpha",
     ]
 )