bug 764671 - Stop uploading symbols for test programs/libs to the symbol server. r=nthomas

Makefile.in

@@ -166,6 +166,7 @@ endif
 	OBJCOPY="$(OBJCOPY)" \
 	$(PYTHON) $(topsrcdir)/toolkit/crashreporter/tools/symbolstore.py \
 	  $(MAKE_SYM_STORE_ARGS)                                          \
+	  --exclude="*test*" --exclude="*Test*"                           \
 	  $(foreach dir,$(SYM_STORE_SOURCE_DIRS),-s $(dir))               \
 	  $(DUMP_SYMS_BIN)                                                \
 	  $(DIST)/crashreporter-symbols                                   \

toolkit/crashreporter/Makefile.in

@@ -98,3 +98,6 @@ endif
 include $(topsrcdir)/config/config.mk
 include $(topsrcdir)/ipc/chromium/chromium-config.mk
 include $(topsrcdir)/config/rules.mk
+
+check::
+	$(PYTHON) $(srcdir)/tools/unit-symbolstore.py

toolkit/crashreporter/tools/symbolstore.py

@@ -26,6 +26,7 @@ import os
 import re
 import shutil
 import textwrap
+import fnmatch
 from subprocess import call, Popen, PIPE, STDOUT
 from optparse import OptionParser
 
@@ -390,7 +391,12 @@ class Dumper:
     ProcessDir.  Instead, call GetPlatformSpecificDumper to
     get an instance of a subclass."""
     def __init__(self, dump_syms, symbol_path,
-                 archs=None, srcdirs=None, copy_debug=False, vcsinfo=False, srcsrv=False):
+                 archs=None,
+                 srcdirs=None,
+                 copy_debug=False,
+                 vcsinfo=False,
+                 srcsrv=False,
+                 exclude=[]):
         # popen likes absolute paths, at least on windows
         self.dump_syms = os.path.abspath(dump_syms)
         self.symbol_path = symbol_path
@@ -406,10 +412,11 @@ class Dumper:
         self.copy_debug = copy_debug
         self.vcsinfo = vcsinfo
         self.srcsrv = srcsrv
+        self.exclude = exclude[:]
 
     # subclasses override this
     def ShouldProcess(self, file):
-        return False
+        return not any(fnmatch.fnmatch(os.path.basename(file), exclude) for exclude in self.exclude)
 
     # and can override this
     def ShouldSkipDir(self, dir):
@@ -545,6 +552,8 @@ class Dumper_Win32(Dumper):
     def ShouldProcess(self, file):
         """This function will allow processing of pdb files that have dll
         or exe files with the same base name next to them."""
+        if not Dumper.ShouldProcess(self, file):
+            return False
         if file.endswith(".pdb"):
             (path,ext) = os.path.splitext(file)
             if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll"):
@@ -616,6 +625,8 @@ class Dumper_Linux(Dumper):
         executable, or end with the .so extension, and additionally
         file(1) reports as being ELF files.  It expects to find the file
         command in PATH."""
+        if not Dumper.ShouldProcess(self, file):
+            return False
         if file.endswith(".so") or os.access(file, os.X_OK):
             return self.RunFileCommand(file).startswith("ELF")
         return False
@@ -654,6 +665,8 @@ class Dumper_Solaris(Dumper):
         executable, or end with the .so extension, and additionally
         file(1) reports as being ELF files.  It expects to find the file
         command in PATH."""
+        if not Dumper.ShouldProcess(self, file):
+            return False
         if file.endswith(".so") or os.access(file, os.X_OK):
             return self.RunFileCommand(file).startswith("ELF")
         return False
@@ -664,6 +677,8 @@ class Dumper_Mac(Dumper):
         executable, or end with the .dylib extension, and additionally
         file(1) reports as being Mach-O files.  It expects to find the file
         command in PATH."""
+        if not Dumper.ShouldProcess(self, file):
+            return False
         if file.endswith(".dylib") or os.access(file, os.X_OK):
             return self.RunFileCommand(file).startswith("Mach-O")
         return False
@@ -736,6 +751,9 @@ def main():
     parser.add_option("-i", "--source-index",
                       action="store_true", dest="srcsrv", default=False,
                       help="Add source index information to debug files, making them suitable for use in a source server.")
+    parser.add_option("-x", "--exclude",
+                      action="append", dest="exclude", default=[], metavar="PATTERN",
+                      help="Skip processing files matching PATTERN.")
     (options, args) = parser.parse_args()
     
     #check to see if the pdbstr.exe exists
@@ -755,7 +773,8 @@ def main():
                                        archs=options.archs,
                                        srcdirs=options.srcdir,
                                        vcsinfo=options.vcsinfo,
-                                       srcsrv=options.srcsrv)
+                                       srcsrv=options.srcsrv,
+                                       exclude=options.exclude)
     for arg in args[2:]:
         dumper.Process(arg)
 

toolkit/crashreporter/tools/unit-symbolstore.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os, tempfile, unittest, shutil, struct, platform
+import symbolstore
+
+# Some simple functions to mock out files that the platform-specific dumpers will accept.
+# dump_syms itself will not be run (we mock that call out), but we can't override
+# the ShouldProcessFile method since we actually want to test that.
+def write_elf(filename):
+    open(filename, "wb").write(struct.pack("<7B45x", 0x7f, ord("E"), ord("L"), ord("F"), 1, 1, 1))
+
+def write_macho(filename):
+    open(filename, "wb").write(struct.pack("<I28x", 0xfeedface))
+
+def write_pdb(filename):
+    open(filename, "w").write("aaa")
+    # write out a fake DLL too
+    open(os.path.splitext(filename)[0] + ".dll", "w").write("aaa")
+
+writer = {'Windows': write_pdb,
+          'Microsoft': write_pdb,
+          'Linux': write_elf,
+          'Sunos5': write_elf,
+          'Darwin': write_macho}[platform.system()]
+extension = {'Windows': ".pdb",
+             'Microsoft': ".pdb",
+             'Linux': ".so",
+             'Sunos5': ".so",
+             'Darwin': ".dylib"}[platform.system()]
+
+def add_extension(files):
+    return [f + extension for f in files]
+
+class TestExclude(unittest.TestCase):
+    """
+    Test that passing filenames to exclude from processing works.
+    """
+    def setUp(self):
+        self.test_dir = tempfile.mkdtemp()
+        if not self.test_dir.endswith("/"):
+            self.test_dir += "/"
+        
+    def tearDown(self):
+        shutil.rmtree(self.test_dir)
+
+    def add_test_files(self, files):
+        for f in files:
+            f = os.path.join(self.test_dir, f)
+            d = os.path.dirname(f)
+            if d and not os.path.exists(d):
+                os.makedirs(d)
+            writer(f)
+
+    def test_exclude_wildcard(self):
+        """
+        Test that using an exclude list with a wildcard pattern works.
+        """
+        processed = []
+        def mock_process_file(filename):
+            processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
+            return True
+        self.add_test_files(add_extension(["foo", "bar", "abc/xyz", "abc/fooxyz", "def/asdf", "def/xyzfoo"]))
+        d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
+                                                  symbol_path="symbol_path",
+                                                  exclude=["*foo*"])
+        d.ProcessFile = mock_process_file
+        self.assertTrue(d.Process(self.test_dir))
+        processed.sort()
+        expected = add_extension(["bar", "abc/xyz", "def/asdf"])
+        expected.sort()
+        self.assertEqual(processed, expected)
+
+    def test_exclude_filenames(self):
+        """
+        Test that excluding a filename without a wildcard works.
+        """
+        processed = []
+        def mock_process_file(filename):
+            processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
+            return True
+        self.add_test_files(add_extension(["foo", "bar", "abc/foo", "abc/bar", "def/foo", "def/bar"]))
+        d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
+                                                  symbol_path="symbol_path",
+                                                  exclude=add_extension(["foo"]))
+        d.ProcessFile = mock_process_file
+        self.assertTrue(d.Process(self.test_dir))
+        processed.sort()
+        expected = add_extension(["bar", "abc/bar", "def/bar"])
+        expected.sort()
+        self.assertEqual(processed, expected)
+
+if __name__ == '__main__':
+  unittest.main()