Update run_tests to generate xml reports for CI (#530)

* Update run_tests to generate xml reports * Setup CI configs for nightly builds running tests Limit number of tests
2018-05-01 15:29:56 -04:00 · 2018-05-01 15:29:56 -04:00 · e2c4005583
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -75,6 +75,51 @@ jobs:
      - store_artifacts:
          path: libchromiumcontent-static.tar.bz2

+  libchromiumcontent-linux-x64-test:
+    docker:
+      - image: electronbuilds/libchromiumcontent:0.0.4
+        environment:
+          TARGET_ARCH: x64
+          COMPONENT: static_library
+          DISPLAY: ':99.0'
+    resource_class: 2xlarge
+    steps:
+      - checkout
+      - run:
+          name: Bootstrap
+          command: script/bootstrap
+      - run:
+          name: Update
+          command: script/update --clean -t $TARGET_ARCH
+      - run:
+          name: Build static library
+          command: script/build -t $TARGET_ARCH -c $COMPONENT
+      - run:
+          name: Build FFmpeg
+          command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Build Tests
+          command: script/build -t $TARGET_ARCH -c tests
+      - run:
+          name: Create distribution
+          command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
+          no_output_timeout: 20m
+      - run:
+          name: Setup for headless testing
+          command: |
+            XVFB=/usr/bin/Xvfb
+            XVFBARGS="$DISPLAY -ac -screen 0 1024x768x16 +extension RANDR"
+            /sbin/start-stop-daemon --start --quiet --background --exec $XVFB -- $XVFBARGS
+      - run:
+          name: Run Tests
+          command: |
+            set +e
+            script/run_tests --generate_report -t $TARGET_ARCH
+      - store_test_results:
+          path: test_reports
+      - store_artifacts:
+          path: test_reports
+
  libchromiumcontent-linux-ia32-shared:
    docker:
      - image: electronbuilds/libchromiumcontent:0.0.4
@ -150,6 +195,48 @@ jobs:
      - store_artifacts:
          path: libchromiumcontent-static.tar.bz2

+  libchromiumcontent-linux-ia32-test:
+    docker:
+      - image: electronbuilds/libchromiumcontent:0.0.4
+        environment:
+          TARGET_ARCH: ia32
+          COMPONENT: static_library
+          DISPLAY: ':99.0'
+    resource_class: 2xlarge
+    steps:
+      - checkout
+      - run:
+          name: Bootstrap
+          command: script/bootstrap
+      - run:
+          name: Update
+          command: script/update --clean -t $TARGET_ARCH
+      - run:
+          name: Build static library
+          command: script/build -t $TARGET_ARCH -c $COMPONENT
+      - run:
+          name: Build FFmpeg
+          command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Create distribution
+          command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
+          no_output_timeout: 20m
+      - run:
+          name: Setup for headless testing
+          command: |
+            XVFB=/usr/bin/Xvfb
+            XVFBARGS="$DISPLAY -ac -screen 0 1024x768x16 +extension RANDR"
+            /sbin/start-stop-daemon --start --quiet --background --exec $XVFB -- $XVFBARGS
+      - run:
+          name: Run Tests
+          command: |
+            set +e
+            script/run_tests --generate_report -t $TARGET_ARCH
+      - store_test_results:
+          path: test_reports
+      - store_artifacts:
+          path: test_reports          
+
  libchromiumcontent-linux-arm-shared:
    docker:
      - image: electronbuilds/libchromiumcontent:0.0.4
@ -410,3 +497,14 @@ workflows:
    jobs:
      - libchromiumcontent-linux-arm64-shared
      - libchromiumcontent-linux-arm64-static
+  nightly-test:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - libchromiumcontent-linux-x64-test
+      - libchromiumcontent-linux-ia32-test
--- a/appveyor.yml
+++ b/appveyor.yml
@ -0,0 +1,50 @@
+version: 1.0.{build}
+build_cloud: libcc-20
+image: libcc-20-vs2017
+clone_folder: c:\build
+environment:
+  matrix:
+  - TARGET_ARCH: ia32
+    COMPONENT: shared_library
+  - TARGET_ARCH: x64
+    COMPONENT: shared_library
+  - TARGET_ARCH: ia32
+    COMPONENT: static_library
+  - TARGET_ARCH: x64
+    COMPONENT: static_library
+build_script:
+- ps: >-
+    if(($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME -split "/")[0] -eq ($env:APPVEYOR_REPO_NAME -split "/")[0]) {
+      Write-warning "Skipping PR build for branch"; Exit-AppveyorBuild
+    } else {
+      if($env:APPVEYOR_SCHEDULED_BUILD -eq 'True')  {
+        script\cibuild.ps1 -buildTests
+      } else {
+        script\cibuild.ps1
+      }
+      if ($? -ne 'True') {
+        throw "Build failed with exit code $?"
+      } else {
+        "Build succeeded."
+      }
+    }
+test_script:
+- ps: >-
+    if($env:COMPONENT -eq 'static_library' -And $env:APPVEYOR_SCHEDULED_BUILD -eq 'True')  {
+      python script\run_tests --generate_report -t $env:TARGET_ARCH
+      npm install -g junit-merge
+      junit-merge -d test_reports
+      # upload results to AppVeyor
+      $wc = New-Object 'System.Net.WebClient'
+      $wc.UploadFile("https://windows-ci.electronjs.org/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\merged-test-results.xml))
+    } else {
+      Write-Output "Skipping tests for shared_library build"
+    }
+
+artifacts:
+- path: libchromiumcontent.zip
+  name: libchromiumcontent.zip
+- path: libchromiumcontent-static.zip
+  name: libchromiumcontent-static.zip
+- path: 'merged-test-results.xml'
+  name: tests
--- a/chromiumcontent/BUILD.gn
+++ b/chromiumcontent/BUILD.gn
@ -357,57 +357,57 @@ if (is_electron_build && !is_component_build) {

    deps = [
      "//base:base_unittests",
-      "//cc:cc_unittests",
-      "//cc/blink:cc_blink_unittests",
-      "//content/test:content_browsertests",
-      "//content/test:content_unittests",
-      "//crypto:crypto_unittests",
-      "//device:device_unittests",
-      "//gin:gin_unittests",
-      "//gpu:gpu_unittests",
-      "//gpu/ipc/service:gpu_ipc_service_unittests",
-      "//ipc:ipc_tests",
-      "//media:media_unittests",
-      "//media/midi:midi_unittests",
-      "//media/mojo:media_mojo_unittests",
-      "//media/capture:capture_unittests",
-      "//mojo/common:mojo_common_unittests",
-      "//mojo/edk/system:mojo_system_unittests",
-      "//mojo/edk/test:mojo_public_bindings_unittests",
-      "//mojo/edk/test:mojo_public_system_unittests",
-      "//net:net_unittests",
-      "//ppapi:ppapi_unittests",
-      "//printing:printing_unittests",
-      "//skia:skia_unittests",
-      "//storage:storage_unittests",
-      "//sql:sql_unittests",
-      "//third_party/angle/src/tests:angle_unittests",
-      "//third_party/leveldatabase:env_chromium_unittests",
-      "//third_party/WebKit/public:test_support",
-      #"//third_party/WebKit/public:all_blink", #TODO: Fix build
-      "//tools/gn:gn_unittests",
-      "//ui/base:ui_base_unittests",
-      #"//ui/compositor:compositor_unittests", # these are failing in chromium
-      "//ui/display:display_unittests",
-      "//ui/events:events_unittests",
-      "//ui/gl:gl_unittests",
-      "//url:url_unittests",
-      "//url/ipc:url_ipc_unittests",
-      "//v8/test/unittests:unittests",
+      # "//cc:cc_unittests",
+      # "//cc/blink:cc_blink_unittests",
+      # "//content/test:content_browsertests",
+      # "//content/test:content_unittests",
+      # "//crypto:crypto_unittests",
+      # "//device:device_unittests",
+      # "//gin:gin_unittests",
+      # "//gpu:gpu_unittests",
+      # "//gpu/ipc/service:gpu_ipc_service_unittests",
+      # "//ipc:ipc_tests",
+      # "//media:media_unittests",
+      # "//media/midi:midi_unittests",
+      # "//media/mojo:media_mojo_unittests",
+      # "//media/capture:capture_unittests",
+      # "//mojo/common:mojo_common_unittests",
+      # "//mojo/edk/system:mojo_system_unittests",
+      # "//mojo/edk/test:mojo_public_bindings_unittests",
+      # "//mojo/edk/test:mojo_public_system_unittests",
+      # "//net:net_unittests",
+      # "//ppapi:ppapi_unittests",
+      # "//printing:printing_unittests",
+      # "//skia:skia_unittests",
+      # "//storage:storage_unittests",
+      # "//sql:sql_unittests",
+      # "//third_party/angle/src/tests:angle_unittests",
+      # "//third_party/leveldatabase:env_chromium_unittests",
+      # "//third_party/WebKit/public:test_support",
+      # "//third_party/WebKit/public:all_blink", #TODO: Fix build
+      # "//tools/gn:gn_unittests",
+      # "//ui/base:ui_base_unittests",
+      # "//ui/compositor:compositor_unittests", # these are failing in chromium
+      # "//ui/display:display_unittests",
+      # "//ui/events:events_unittests",
+      # "//ui/gl:gl_unittests",
+      # "//url:url_unittests",
+      # "//url/ipc:url_ipc_unittests",
+      # "//v8/test/unittests:unittests",
    ]

-    if (is_linux) {
-      deps += [
-        "//net:disk_cache_memory_test",
-        "//sandbox/linux:sandbox_linux_unittests",
-      ]
-
-      if (use_dbus) {
-        deps += [
-          "//dbus:dbus_unittests",
-        ]
-      }
-    }
+    # if (is_linux) {
+    #   deps += [
+    #     "//net:disk_cache_memory_test",
+    #     "//sandbox/linux:sandbox_linux_unittests",
+    #   ]
+    #
+    #   if (use_dbus) {
+    #     deps += [
+    #       "//dbus:dbus_unittests",
+    #     ]
+    #   }
+    # }
  }

 }
--- a/script/cibuild
+++ b/script/cibuild
@ -19,6 +19,7 @@ def main():
  args = parse_args()
  skip_upload = args.skip_upload
  use_sccache = args.use_sccache
+  build_tests = args.build_tests

  if (not skip_upload and
     not 'LIBCHROMIUMCONTENT_S3_ACCESS_KEY' in os.environ and
@ -33,20 +34,23 @@ def main():
  # Use by gclient to distinguish bot builds
  os.environ['CHROME_HEADLESS'] = '1'

+  component = None
  if 'TARGET_ARCH' in os.environ:
-    args = ['-t', os.environ['TARGET_ARCH']]
-    return run_ci(args, skip_upload, use_sccache, os.environ['COMPONENT'])
-
-  if sys.platform in ['win32', 'cygwin']:
-    return (run_ci(['-t', 'x64'], skip_upload, use_sccache) or
-            run_ci(['-t', 'ia32'], skip_upload, use_sccache))
+    archs = [os.environ['TARGET_ARCH']]
+    component = os.environ['COMPONENT']
+  elif sys.platform in ['win32', 'cygwin']:
+    archs = ['x64', 'ia32']
  elif sys.platform == 'linux2':
-    return (run_ci(['-t', 'x64'], skip_upload, use_sccache) or
-            run_ci(['-t', 'ia32'], skip_upload, use_sccache) or
-            run_ci(['-t', 'arm'], skip_upload, use_sccache) or
-            run_ci(['-t', 'arm64'], skip_upload, use_sccache))
+    archs = ['x64', 'ia32', 'arm', 'arm64']
  else:
-    return run_ci([], skip_upload, use_sccache)
+    archs = [None]
+
+  ci_results = True
+  for arch in archs:
+    if not run_ci(arch, skip_upload, use_sccache, build_tests, component):
+      return False
+
+    return True

 def parse_args():
  parser = argparse.ArgumentParser(description='Run CI build')
@ -54,6 +58,8 @@ def parse_args():
                      help='Skip uploading to S3')
  parser.add_argument('--use_sccache', action='store_true',
                      help='Use sccache binary stored in with the libcc repo.')
+  parser.add_argument('--build_tests', action='store_true',
+                      help='Build tests as part of build.')
  return parser.parse_args()

 def copy_to_environment(credentials_file):
@ -75,7 +81,10 @@ def os_version():
    return platform.platform()


-def run_ci(args, skip_upload, use_sccache, component=None):
+def run_ci(target_arch, skip_upload, use_sccache, build_tests, component=None):
+  args = []
+  if target_arch is not None:
+    args += ['-t', target_arch]
  build_args = []
  component_args = []
  update_args = ['--clean'] + args
@ -84,8 +93,9 @@ def run_ci(args, skip_upload, use_sccache, component=None):
  if component is not None:
    component_args = ['-c', component]
    build_args += component_args + ['ffmpeg']
+    if build_tests:
+      build_args += ['tests']
  if sys.platform in ['win32', 'cygwin']:
-    target_arch = args[1]
    # Set build env for VS.
    if target_arch == 'x64':
      vs_env = get_vs_env('14.0', 'x86_amd64')
--- a/script/cibuild.ps1
+++ b/script/cibuild.ps1
@ -1,4 +1,4 @@
-param([switch]$skipUpload, [switch]$useSccache)
+param([switch]$skipUpload, [switch]$useSccache, [switch]$buildTests)
 function Run-Command([scriptblock]$Command, [switch]$Fatal, [switch]$Quiet) {
  $output = ""
  try {
@ -41,5 +41,9 @@ if ($skipUpload) {
 if ($useSccache) {
  $CommandLine += " --use_sccache"
 }
+
+if ($buildTests) {
+  $CommandLine += " --build_tests"
+}
 $CICommand = [ScriptBlock]::Create($CommandLine)
 Run-Command -Fatal $CICommand
--- a/script/run_tests
+++ b/script/run_tests
@ -6,9 +6,12 @@ import platform
 import re
 import sys
 import subprocess
-import yaml

-from lib.config import get_output_dir
+from lib.config import get_output_dir, VENDOR_DIR
+
+PYYAML_LIB_DIR = os.path.join(VENDOR_DIR, 'pyyaml', 'lib')
+sys.path.append(PYYAML_LIB_DIR)
+import yaml

 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 if sys.platform in ['win32', 'cygwin']:
@ -36,7 +39,7 @@ def GetTestsToRun(args):
        print 'Invalid test target name: ' + test_name
  return tests

-def RunTests(binary_tests):
+def RunTests(binary_tests, generate_report):
  if binary_tests:
    disabled_tests_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'disabled-tests.yaml')
    disabled_tests = {}
@ -51,6 +54,9 @@ def RunTests(binary_tests):
      if disabled_tests and test_name in disabled_tests and disabled_tests[test_name]:
        exclusion_filter = '-' + ':'.join(disabled_tests[test_name])
        test_args.append('--gtest_filter=%s' % exclusion_filter)
+      # Add reporting path if enabled
+      if generate_report:
+        test_args.append("--gtest_output=xml:" + get_output_file_path(test_name))
      results.append((os.path.basename(test),
                      subprocess.call(test_args) == 0))
    failed = [test
@ -68,14 +74,18 @@ def RunTests(binary_tests):
    print 'Nothing to test - no tests found!'
  return 0

+def get_output_file_path(test_name):
+  return os.path.join(SOURCE_ROOT, 'test_reports', test_name + os.path.extsep + 'xml')
+
 def main():
  parser = argparse.ArgumentParser(description='Run Chromium tests')
  parser.add_argument('-t', '--target_arch', default='x64', help='x64 or ia32')
  parser.add_argument('--only', nargs='+', help='Names of the test targets to run. For example: base_unittests url_unittests')
  parser.add_argument('--exclude', nargs='+', help='Names of the test targets to exclude. For example: base_unittests url_unittests')
-  tests = GetTestsToRun(parser.parse_args())
-  return RunTests(tests)
+  parser.add_argument('--generate_report', action='store_true', help='Generate xml reports for test runs. Mainly for CI.')
+  args = parser.parse_args()
+  tests = GetTestsToRun(args)
+  return RunTests(tests, args.generate_report)

 if __name__ == '__main__':
  sys.exit(main())
-
--- a/vsts.yml
+++ b/vsts.yml
@ -28,6 +28,7 @@ phases:

  steps:
  - bash: |
+     set -e
     if [[ -z "${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER}" ]] || "${SYSTEM_PULLREQUEST_ISFORK}" == "True"; then
       echo "##vso[task.setvariable variable=gitcommit]$BUILD_SOURCEVERSION"
     else
@ -42,11 +43,13 @@ phases:
    name: Get_commit

  - bash: |
+     set -e
     echo "===Bootstrapping===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     script/bootstrap > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
    name: Bootstrap

  - bash: |
+     set -e
     echo "===Updating for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     if [ "${COMPONENT}" == "shared_library" ]; then
       script/update --clean -t $TARGET_ARCH --use-bundled-sccache > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
@ -56,6 +59,7 @@ phases:
    name: Update

  - bash: |
+     set -e
     if [ "${COMPONENT}" == "shared_library" ]; then
       script/sccache --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
       script/sccache -s > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
@ -66,10 +70,15 @@ phases:
     script/build -t $TARGET_ARCH -c ffmpeg > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     if [ "${COMPONENT}" == "shared_library" ]; then
       script/sccache --stop-server > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     else
+       if [ "${BUILD_REASON}" == "Schedule" ]; then
+         script/build -t $TARGET_ARCH -c tests > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+       fi
     fi
    name: Build

  - bash: |
+     set -e
     echo "===Create $COMPONENT distribution for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     script/create-dist -t $TARGET_ARCH -c $COMPONENT > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     mkdir s3files
@ -77,6 +86,21 @@ phases:
     mv buildlog.txt s3files
    name: Create_distribution

+  - bash: |
+     set +e
+     echo "===Run tests against $COMPONENT distribution for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     script/run_tests --generate_report > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     cp -r test_reports s3files
+    name: Run_tests
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'Schedule'), eq(variables['COMPONENT'], 'static_library'))
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '*.xml'
+      searchFolder: '$(System.DefaultWorkingDirectory)/test_reports'
+      testRunTitle: 'Libchromiumcontent Tests'
+    condition: and(always(), eq(variables['Build.Reason'], 'Schedule'), eq(variables['COMPONENT'], 'static_library'))
+
  - task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1
    inputs:
      awsCredentials: 'Libchromium Content S3'