ci: run unittests on clean Chromium (#15125)

* ci: run unittests on clean Chromium * test: add '--run-only-disabled-tests' flag to "script/native-tests.py" * ci: add a job to run only disabled unittests * test: use a different GN config for the native tests * test: enable blink_common_unittests * test: disable WeakPtrDeathTest*
2018-10-22 22:12:19 +02:00 · 2018-10-22 22:12:19 +02:00 · 5e199152e2
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -42,13 +42,13 @@ env-stack-dumping: &env-stack-dumping
  ELECTRON_ENABLE_STACK_DUMPING: '1'

 env-browsertests: &env-browsertests
-  GN_EXTRA_ARGS: 'is_component_ffmpeg = false'
-  BUILD_TARGET: electron:chromium_browsertests
+  GN_CONFIG: //electron/build/args/native_tests.gn
+  BUILD_TARGET: electron/spec:chromium_browsertests
  TESTS_CONFIG: src/electron/spec/configs/browsertests.yml

 env-unittests: &env-unittests
-  GN_EXTRA_ARGS: 'is_component_ffmpeg = false'
-  BUILD_TARGET: electron:chromium_unittests
+  GN_CONFIG: //electron/build/args/native_tests.gn
+  BUILD_TARGET: electron/spec:chromium_unittests
  TESTS_CONFIG: src/electron/spec/configs/unittests.yml

 # Build targets options.
@ -598,7 +598,8 @@ steps-native-tests: &steps-native-tests
          python src/electron/script/native-tests.py run \
            --config $TESTS_CONFIG \
            --tests-dir src/out/Default \
-            --output-dir test_results
+            --output-dir test_results  \
+            $TESTS_ARGS

    - store_artifacts:
        path: test_results
@ -723,6 +724,12 @@ jobs:
    <<: *machine-linux-2xlarge
    <<: *steps-checkout

+  linux-x64-checkout-no-patches:
+    <<: *machine-linux-2xlarge
+    environment:
+      GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False'
+    <<: *steps-checkout
+
  # Layer 2: Builds.
  linux-x64-debug:
    <<: *machine-linux-2xlarge
@ -956,7 +963,23 @@ jobs:
    <<: *machine-linux-2xlarge
    environment:
      <<: *env-unittests
-      <<: *env-testing-build
+      <<: *env-enable-sccache
+      <<: *env-headless-testing
+    <<: *steps-native-tests
+
+  linux-x64-disabled-unittests:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-unittests
+      <<: *env-enable-sccache
+      <<: *env-headless-testing
+      TESTS_ARGS: '--run-only-disabled-tests'
+    <<: *steps-native-tests
+
+  linux-x64-chromium-unittests:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-unittests
      <<: *env-enable-sccache
      <<: *env-headless-testing
    <<: *steps-native-tests
@ -1234,3 +1257,13 @@ workflows:
      - linux-x64-unittests:
          requires:
            - linux-x64-checkout
+
+      - linux-x64-disabled-unittests:
+          requires:
+            - linux-x64-checkout
+
+      - linux-x64-checkout-no-patches
+
+      - linux-x64-chromium-unittests:
+          requires:
+            - linux-x64-checkout-no-patches
--- a/BUILD.gn
+++ b/BUILD.gn
@ -810,75 +810,6 @@ if (is_mac) {
  }
 }

-group("electron_tests") {
-  testonly = true
-
-  deps = [
-    ":chromium_browsertests",
-    ":chromium_unittests",
-  ]
-}
-
-group("chromium_unittests") {
-  testonly = true
-
-  deps = [
-    "//base:base_unittests",
-    "//cc:cc_unittests",
-    "//content/test:content_unittests",
-    "//crypto:crypto_unittests",
-    "//device:device_unittests",
-    "//gin:gin_unittests",
-    "//gpu:gpu_unittests",
-    "//ipc:ipc_tests",
-    "//media:media_unittests",
-    "//media/capture:capture_unittests",
-    "//media/midi:midi_unittests",
-    "//media/mojo:media_mojo_unittests",
-    "//mojo:mojo_unittests",
-    "//net:net_unittests",
-    "//ppapi:ppapi_unittests",
-    "//skia:skia_unittests",
-    "//sql:sql_unittests",
-    "//storage:storage_unittests",
-    "//third_party/angle/src/tests:angle_unittests",
-    "//third_party/blink/public:all_blink",
-    "//third_party/blink/public:test_support",
-    "//third_party/leveldatabase:env_chromium_unittests",
-    "//ui/base:ui_base_unittests",
-    "//ui/compositor:compositor_unittests",
-    "//ui/display:display_unittests",
-    "//ui/events:events_unittests",
-    "//ui/gl:gl_unittests",
-    "//url:url_unittests",
-    "//url/ipc:url_ipc_unittests",
-    "//v8/test/unittests:unittests",
-  ]
-
-  if (enable_basic_printing) {
-    deps += [ "//printing:printing_unittests" ]
-  }
-
-  if (is_linux) {
-    deps += [
-      "//net:disk_cache_memory_test",
-      "//sandbox/linux:sandbox_linux_unittests",
-    ]
-
-    if (use_dbus) {
-      deps += [ "//dbus:dbus_unittests" ]
-    }
-  }
-}
-
-group("chromium_browsertests") {
-  testonly = true
-
-  deps = [
-    "//content/test:content_browsertests",
-  ]
-}
-
 template("dist_zip") {
  _runtime_deps_target = "${target_name}__deps"
  _runtime_deps_file =
--- a/build/args/native_tests.gn
+++ b/build/args/native_tests.gn
@ -0,0 +1,8 @@
+root_extra_deps = [ "//electron/spec" ]
+
+dcheck_always_on = true
+is_debug = false
+is_component_build = false
+is_component_ffmpeg = false
+symbol_level = 1
+use_jumbo_build = true
--- a/script/lib/native_tests.py
+++ b/script/lib/native_tests.py
@ -87,7 +87,8 @@ class TestsList():
    supported_binaries = filter(self.__platform_supports, all_binaries)
    return supported_binaries

-  def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY):
+  def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY,
+      run_only_disabled_tests=False):
    # Don't run anything twice.
    binaries = set(binaries)

@ -105,14 +106,19 @@ class TestsList():
                binary_name, Platform.get_current()))

    suite_returncode = sum(
-        [self.__run(binary, output_dir, verbosity) for binary in binaries])
+        [self.__run(binary, output_dir, verbosity, run_only_disabled_tests)
+        for binary in binaries])
    return suite_returncode

-  def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY):
-    return self.run([binary_name], output_dir, verbosity)
+  def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY,
+      run_only_disabled_tests=False):
+    return self.run([binary_name], output_dir, verbosity,
+                    run_only_disabled_tests)

-  def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY):
-    return self.run(self.get_for_current_platform(), output_dir, verbosity)
+  def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY,
+      run_only_disabled_tests=False):
+    return self.run(self.get_for_current_platform(), output_dir, verbosity,
+                    run_only_disabled_tests)

  @staticmethod
  def __get_tests_list(config_path):
@ -169,7 +175,7 @@ class TestsList():

    binary_name = data_item.keys()[0]
    test_data = {
-      'excluded_tests': None,
+      'excluded_tests': [],
      'platforms': Platform.get_all()
    }

@ -193,16 +199,26 @@ class TestsList():

    return (binary_name, test_data)

-  def __run(self, binary_name, output_dir, verbosity):
+  def __run(self, binary_name, output_dir, verbosity,
+      run_only_disabled_tests):
    binary_path = os.path.join(self.tests_dir, binary_name)
    test_binary = TestBinary(binary_path)

    test_data = self.tests[binary_name]
+    included_tests = []
    excluded_tests = test_data['excluded_tests']

+    if run_only_disabled_tests and len(excluded_tests) == 0:
+      # There is nothing to run.
+      return 0
+
+    if run_only_disabled_tests:
+      included_tests, excluded_tests = excluded_tests, included_tests
+
    output_file_path = TestsList.__get_output_path(binary_name, output_dir)

-    return test_binary.run(excluded_tests=excluded_tests,
+    return test_binary.run(included_tests=included_tests,
+                           excluded_tests=excluded_tests,
                           output_file_path=output_file_path,
                           verbosity=verbosity)

@ -221,9 +237,10 @@ class TestBinary():
  def __init__(self, binary_path):
    self.binary_path = binary_path

-  def run(self, excluded_tests=None, output_file_path=None,
-      verbosity=Verbosity.CHATTY):
-    gtest_filter = TestBinary.__get_gtest_filter(excluded_tests)
+  def run(self, included_tests=None, excluded_tests=None,
+      output_file_path=None, verbosity=Verbosity.CHATTY):
+    gtest_filter = TestBinary.__get_gtest_filter(included_tests,
+                                                 excluded_tests)
    gtest_output = TestBinary.__get_gtest_output(output_file_path)

    args = [self.binary_path, gtest_filter, gtest_output]
@ -241,12 +258,12 @@ class TestBinary():
    return returncode

  @staticmethod
-  def __get_gtest_filter(excluded_tests):
-    gtest_filter = ""
-    if excluded_tests is not None and len(excluded_tests) > 0:
-      excluded_tests_string = TestBinary.__format_excluded_tests(
-          excluded_tests)
-      gtest_filter = "--gtest_filter={}".format(excluded_tests_string)
+  def __get_gtest_filter(included_tests, excluded_tests):
+    included_tests_string = TestBinary.__list_tests(included_tests)
+    excluded_tests_string = TestBinary.__list_tests(excluded_tests)
+
+    gtest_filter = "--gtest_filter={}-{}".format(included_tests_string,
+                                                 excluded_tests_string)
    return gtest_filter

  @staticmethod
@ -258,8 +275,10 @@ class TestBinary():
    return gtest_output

  @staticmethod
-  def __format_excluded_tests(excluded_tests):
-    return "-" + ":".join(excluded_tests)
+  def __list_tests(tests):
+    if tests is None:
+      return ''
+    return ':'.join(tests)

  @staticmethod
  def __get_stdout_and_stderr(verbosity):
--- a/script/native-tests.py
+++ b/script/native-tests.py
@ -25,6 +25,9 @@ def parse_args():
                      help='binaries to run')
  parser.add_argument('-c', '--config', required=True,
                      help='path to a tests config')
+  parser.add_argument('--run-only-disabled-tests',
+                      action='store_true', default=False,
+                      help='if disabled tests should be run')
  parser.add_argument('-t', '--tests-dir', required=False,
                      help='path to a directory with test binaries')
  parser.add_argument('-o', '--output-dir', required=False,
@ -82,9 +85,11 @@ def main():

  if args.command == Command.RUN:
    if args.binary is not None:
-      return tests_list.run(args.binary, args.output_dir, args.verbosity)
+      return tests_list.run(args.binary, args.output_dir, args.verbosity,
+                            args.run_only_disabled_tests)
    else:
-      return tests_list.run_all(args.output_dir, args.verbosity)
+      return tests_list.run_all(args.output_dir, args.verbosity,
+                                args.run_only_disabled_tests)

  assert False, "unexpected command '{}'".format(args.command)

--- a/spec/BUILD.gn
+++ b/spec/BUILD.gn
@ -0,0 +1,72 @@
+import("//build/config/features.gni")
+import("//printing/buildflags/buildflags.gni")
+
+group("spec") {
+  testonly = true
+
+  deps = [
+    ":chromium_browsertests",
+    ":chromium_unittests",
+  ]
+}
+
+group("chromium_unittests") {
+  testonly = true
+
+  deps = [
+    "//base:base_unittests",
+    "//cc:cc_unittests",
+    "//content/test:content_unittests",
+    "//crypto:crypto_unittests",
+    "//device:device_unittests",
+    "//gin:gin_unittests",
+    "//gpu:gpu_unittests",
+    "//ipc:ipc_tests",
+    "//media:media_unittests",
+    "//media/capture:capture_unittests",
+    "//media/midi:midi_unittests",
+    "//media/mojo:media_mojo_unittests",
+    "//mojo:mojo_unittests",
+    "//net:net_unittests",
+    "//ppapi:ppapi_unittests",
+    "//printing:printing_unittests",
+    "//skia:skia_unittests",
+    "//sql:sql_unittests",
+    "//storage:storage_unittests",
+    "//third_party/angle/src/tests:angle_unittests",
+    "//third_party/blink/public:all_blink",
+    "//third_party/blink/public:test_support",
+    "//third_party/leveldatabase:env_chromium_unittests",
+    "//ui/base:ui_base_unittests",
+    "//ui/compositor:compositor_unittests",
+    "//ui/display:display_unittests",
+    "//ui/events:events_unittests",
+    "//ui/gl:gl_unittests",
+    "//url:url_unittests",
+    "//url/ipc:url_ipc_unittests",
+    "//v8/test/unittests:unittests",
+  ]
+
+  if (enable_basic_printing) {
+    deps += [ "//printing:printing_unittests" ]
+  }
+
+  if (is_linux) {
+    deps += [
+      "//net:disk_cache_memory_test",
+      "//sandbox/linux:sandbox_linux_unittests",
+    ]
+
+    if (use_dbus) {
+      deps += [ "//dbus:dbus_unittests" ]
+    }
+  }
+}
+
+group("chromium_browsertests") {
+  testonly = true
+
+  deps = [
+    "//content/test:content_browsertests",
+  ]
+}
--- a/spec/configs/unittests.yml
+++ b/spec/configs/unittests.yml
@ -10,6 +10,7 @@ tests:
          - RTLTest*
          - SysStrings*
          - UTFOffsetStringConversionsTest*
+          - WeakPtrDeathTest*
  - cc_unittests
  - cc_blink_unittests
  - content_unittests:
@ -75,8 +76,7 @@ tests:
        # TODO(alexeykuzmin): Should it be fixed?
        - LiveRangeUnitTest*
  # Blink
-# TODO: Enable in Ch68.
-#  - blink_common_unittests
+  - blink_common_unittests
  - blink_heap_unittests
  - blink_platform_unittests:
      disabled: