Bug 1475256 - Add code coverage for android robocop UI tests. r=gbrown

Depends on D4219

Differential Revision: https://phabricator.services.mozilla.com/D4144

--HG--
extra : moz-landing-system : lando

mobile/android/app/build.gradle

@@ -532,6 +532,28 @@ if (project.hasProperty('enable_code_coverage')) {
 
 // Set up code coverage for tests on emulators.
 if (mozconfig.substs.MOZ_JAVA_CODE_COVERAGE) {
+    apply plugin: "jacoco"
+    jacoco {
+        toolVersion = "${project.jacoco_version}"
+    }
+
+    android {
+        jacoco {
+            version = "$jacoco_version"
+        }
+        buildTypes {
+            debug {
+                testCoverageEnabled true
+            }
+        }
+    }
+
+    dependencies {
+        // This is required both in the instrumented application classes and the test classes,
+        // so `api` has to be used instead of `androidTestImplementation`.
+        api "org.jacoco:org.jacoco.agent:$jacoco_version:runtime"
+    }
+
     // Generate tasks to archive compiled classfiles for later use with JaCoCo report generation.
     // One of these tasks is used by `mach android archive-coverage-artifacts`.
     android.applicationVariants.all { variant ->

taskcluster/ci/test/misc.yml

@@ -69,6 +69,7 @@ robocop:
     chunks:
         by-test-platform:
             # android-em-4.3-arm7-api-16/debug -- not run
+            android-em-4.3-arm7-api-16-ccov/debug: 8
             android-em-4.3-arm7-api-16/opt: 4
     loopback-video: true
     e10s: false

taskcluster/ci/test/test-sets.yml

@@ -408,6 +408,7 @@ android-x86-kvm-tests:
 
 android-ccov-tests:
     - geckoview-junit
+    - robocop
 
 devtools-tests:
     - mochitest-devtools-chrome

testing/mochitest/mochitest_options.py

@@ -904,6 +904,18 @@ class AndroidArguments(ArgumentContainer):
                    (eg. /mnt/sdcard/tests or /data/local/tests)",
           "suppress": True,
           }],
+        [["--enable-coverage"],
+         {"action": "store_true",
+          "default": False,
+          "help": "Enable collecting code coverage information when running"
+                  "robocop tests.",
+          }],
+        [["--coverage-output-dir"],
+         {"action": "store",
+          "default": None,
+          "help": "When using --enable-java-coverage, save the code coverage report"
+                  "files to this directory.",
+          }],
     ]
 
     defaults = {
@@ -970,6 +982,21 @@ class AndroidArguments(ArgumentContainer):
                     options.robocopApk)
             options.robocopApk = os.path.abspath(options.robocopApk)
 
+        if options.coverage_output_dir and not options.enable_coverage:
+            parser.error("--coverage-output-dir must be used with --enable-coverage")
+        if options.enable_coverage:
+            if not options.autorun:
+                parser.error(
+                    "--enable-coverage cannot be used with --no-autorun")
+            if not options.coverage_output_dir:
+                parser.error(
+                    "--coverage-output-dir must be specified when using --enable-coverage")
+            parent_dir = os.path.dirname(options.coverage_output_dir)
+            if not os.path.isdir(options.coverage_output_dir):
+                parser.error(
+                    "The directory for the coverage output does not exist: %s" %
+                    parent_dir)
+
         # Disable e10s by default on Android because we don't run Android
         # e10s jobs anywhere yet.
         options.e10s = False

testing/mochitest/runrobocop.py

@@ -418,17 +418,34 @@ class RobocopTestRunner(MochitestDesktop):
         self.setupRemoteProfile()
         self.options.app = "am"
         timeout = None
+
+        testName = test['name'].split('/')[-1].split('.java')[0]
+        if self.options.enable_coverage:
+            remoteCoverageFile = posixpath.join(self.options.remoteTestRoot,
+                                                'robocop-coverage-%s.ec' % testName)
+            coverageFile = os.path.join(self.options.coverage_output_dir,
+                                        'robocop-coverage-%s.ec' % testName)
         if self.options.autorun:
             # This launches a test (using "am instrument") and instructs
             # Fennec to /quit/ the browser (using Robocop:Quit) and to
             # /finish/ all opened activities.
             browserArgs = [
                 "instrument",
+            ]
+
+            if self.options.enable_coverage:
+                browserArgs += [
+                    "-e", "coverage", "true",
+                    "-e", "coverageFile", remoteCoverageFile,
+                ]
+
+            browserArgs += [
                 "-e", "quit_and_finish", "1",
                 "-e", "deviceroot", self.device.test_root,
                 "-e", "class",
-                "org.mozilla.gecko.tests.%s" % test['name'].split('/')[-1].split('.java')[0],
-                "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner"]
+                "org.mozilla.gecko.tests.%s" % testName,
+                "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner",
+            ]
         else:
             # This does not launch a test at all. It launches an activity
             # that starts Fennec and then waits indefinitely, since cat
@@ -465,6 +482,14 @@ class RobocopTestRunner(MochitestDesktop):
                 # terse.
                 if self.options.log_mach is None:
                     self.printDeviceInfo(printLogcat=True)
+            if self.options.enable_coverage:
+                if self.device.is_file(remoteCoverageFile):
+                    self.device.pull(remoteCoverageFile, coverageFile)
+                    self.device.rm(remoteCoverageFile)
+                else:
+                    self.log.warning("Code coverage output not found on remote device: %s" %
+                                     remoteCoverageFile)
+
         except Exception:
             self.log.error(
                 "Automation Error: Exception caught while running tests")