Bug 1700693 - Add wrench test for successful shader compilation. r=jrmuizel

Add a new `test_init` subcommand to wrench that tests for successful
initialization. This can then be used in combination with the
`--precache` flag to ensure that all shaders are compiled
successfully. Run this with and without `--use-unoptimized-shaders` to
ensure that both variants of the shaders compile.

For android we need to ensure that wrench terminates the process when
it completes successfully otherwise it will run forever. We also must
ensure that the test harness stops the old process before relaunching
the activity one in the case of a timeout.

Depends on D109773

Differential Revision: https://phabricator.services.mozilla.com/D109774

gfx/wr/ci-scripts/linux-release-tests.sh

@@ -16,6 +16,10 @@ set -o xtrace
 CARGOFLAGS=${CARGOFLAGS:-""}  # default to empty if not set
 
 pushd wrench
+# Test that all shaders compile successfully.
+python script/headless.py --precache test_init
+python script/headless.py --precache --use-unoptimized-shaders test_init
+
 python script/headless.py reftest
 python script/headless.py rawtest
 python script/headless.py test_invalidation

gfx/wr/ci-scripts/macos-release-tests.sh

@@ -20,6 +20,10 @@ WRENCH_BINARY=${WRENCH_BINARY:-""}
 
 pushd wrench
 
+# Test that all shaders compile successfully.
+python script/headless.py --precache test_init
+python script/headless.py --precache --use-unoptimized-shaders test_init
+
 python script/headless.py reftest
 python script/headless.py test_invalidation
 if [[ -z "${WRENCH_BINARY}" ]]; then

gfx/wr/ci-scripts/windows-tests.cmd

@@ -21,6 +21,13 @@ popd
 pushd wrench
 cargo test --verbose
 if %ERRORLEVEL% NEQ 0 EXIT /b %ERRORLEVEL%
+:: Test that all shaders compile successfully. --precache compiles all shaders
+:: during initialization, therefore if init is successful then the shaders compile.
+cargo run --release -- --angle --precache test_init
+if %ERRORLEVEL% NEQ 0 EXIT /b %ERRORLEVEL%
+cargo run --release -- --angle --precache --use-unoptimized-shaders test_init
+if %ERRORLEVEL% NEQ 0 EXIT /b %ERRORLEVEL%
+
 cargo run --release -- --angle reftest
 if %ERRORLEVEL% NEQ 0 EXIT /b %ERRORLEVEL%
 popd

gfx/wr/webrender/src/device/gl.rs

@@ -1928,7 +1928,12 @@ impl Device {
             self.gl.get_shader_iv(id, gl::COMPILE_STATUS, &mut status);
         }
         if status[0] == 0 {
-            error!("Failed to compile shader: {}\n{}", name, log);
+            let type_str = match shader_type {
+                gl::VERTEX_SHADER => "vertex",
+                gl::FRAGMENT_SHADER => "fragment",
+                _ => panic!("Unexpected shader type {:x}", shader_type),
+            };
+            error!("Failed to compile {} shader: {}\n{}", type_str, name, log);
             #[cfg(debug_assertions)]
             Self::print_shader_errors(source, &log);
             Err(ShaderError::Compilation(name.to_string(), log))
@@ -2241,7 +2246,7 @@ impl Device {
         if build_program {
             // Compile the vertex shader
             let vs_source = info.compute_source(self, ShaderKind::Vertex);
-            let vs_id = match self.compile_shader(&info.base_filename, gl::VERTEX_SHADER, &vs_source) {
+            let vs_id = match self.compile_shader(&info.full_name(), gl::VERTEX_SHADER, &vs_source) {
                     Ok(vs_id) => vs_id,
                     Err(err) => return Err(err),
                 };
@@ -2249,7 +2254,7 @@ impl Device {
             // Compile the fragment shader
             let fs_source = info.compute_source(self, ShaderKind::Fragment);
             let fs_id =
-                match self.compile_shader(&info.base_filename, gl::FRAGMENT_SHADER, &fs_source) {
+                match self.compile_shader(&info.full_name(), gl::FRAGMENT_SHADER, &fs_source) {
                     Ok(fs_id) => fs_id,
                     Err(err) => {
                         self.gl.delete_shader(vs_id);

gfx/wr/wrench/src/args.yaml

@@ -192,3 +192,5 @@ subcommands:
               help: second benchmark file to compare
               required: true
               index: 2
+    - test_init:
+        about: Test for successful initialization then exit immediately

gfx/wr/wrench/src/main.rs

@@ -762,11 +762,19 @@ fn main() {
         let second_filename = subargs.value_of("second_filename").unwrap();
         perf::compare(first_filename, second_filename);
         return;
+    } else if let Some(_) = args.subcommand_matches("test_init") {
+        // Wrench::new() unwraps the Renderer initialization, so if
+        // we reach this point then we have initialized successfully.
+        println!("Initialization successful");
     } else {
         panic!("Should never have gotten here! {:?}", args);
     };
 
     wrench.renderer.deinit();
+
+    // On android force-exit the process otherwise it stays running forever.
+    #[cfg(target_os = "android")]
+    process::exit(0);
 }
 
 fn render<'a>(

testing/mozharness/scripts/android_wrench.py

@@ -7,6 +7,7 @@
 
 from __future__ import absolute_import
 import datetime
+import enum
 import os
 import subprocess
 import sys
@@ -25,6 +26,12 @@ from mozharness.mozilla.testing.android import AndroidMixin
 from mozharness.mozilla.testing.testbase import TestingMixin
 
 
+class TestMode(enum.Enum):
+    OPTIMIZED_SHADER_COMPILATION = 0
+    UNOPTIMIZED_SHADER_COMPILATION = 1
+    REFTEST = 2
+
+
 class AndroidWrench(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin):
     def __init__(self, require_config_file=False):
         # code in BaseScript.__init__ iterates all the properties to attach
@@ -122,30 +129,46 @@ class AndroidWrench(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin):
         end_time = datetime.datetime.now() + datetime.timedelta(seconds=timeout)
         while self.device.process_exist(process_name, timeout=timeout):
             if datetime.datetime.now() > end_time:
+                stop_cmd = [
+                    self.adb_path,
+                    "-s",
+                    self.device_serial,
+                    "shell",
+                    "am",
+                    "force-stop",
+                    process_name,
+                ]
+                subprocess.check_call(stop_cmd)
                 return False
             time.sleep(5)
 
         return True
 
-    def setup_sdcard(self):
+    def setup_sdcard(self, test_mode):
         # Note that we hard-code /sdcard/wrench as the path here, rather than
         # using something like self.device.test_root, because it needs to be
         # kept in sync with the path hard-coded inside the wrench source code.
         self.device.rm("/sdcard/wrench", recursive=True, force=True)
         self.device.mkdir("/sdcard/wrench", parents=True)
-        self.device.push(
+        if test_mode == TestMode.REFTEST:
-            self.query_abs_dirs()["abs_reftests_path"], "/sdcard/wrench/reftests"
+            self.device.push(
-        )
+                self.query_abs_dirs()["abs_reftests_path"], "/sdcard/wrench/reftests"
+            )
         args_file = os.path.join(self.query_abs_dirs()["abs_work_dir"], "wrench_args")
         with open(args_file, "w") as argfile:
             if self.is_emulator:
                 argfile.write("env: WRENCH_REFTEST_CONDITION_EMULATOR=1\n")
             else:
                 argfile.write("env: WRENCH_REFTEST_CONDITION_DEVICE=1\n")
-            argfile.write("reftest")
+            if test_mode == TestMode.OPTIMIZED_SHADER_COMPILATION:
+                argfile.write("--precache test_init")
+            elif test_mode == TestMode.UNOPTIMIZED_SHADER_COMPILATION:
+                argfile.write("--precache --use-unoptimized-shaders test_init")
+            elif test_mode == TestMode.REFTEST:
+                argfile.write("reftest")
         self.device.push(args_file, "/sdcard/wrench/args")
 
-    def run_tests(self):
+    def run_tests(self, timeout):
         self.timed_screenshots(None)
         self.device.launch_application(
             app_name="org.mozilla.wrench",
@@ -153,7 +176,7 @@ class AndroidWrench(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin):
             intent=None,
         )
         self.info("App launched")
-        done = self.wait_until_process_done("org.mozilla.wrench", timeout=60 * 30)
+        done = self.wait_until_process_done("org.mozilla.wrench", timeout=timeout)
         if not done:
             self._errored = True
             self.error("Wrench still running after timeout")
@@ -235,10 +258,25 @@ class AndroidWrench(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin):
         self.info(self.shell_output("getprop"))
         self.info("Installing APK...")
         self.install_apk(self.query_abs_dirs()["abs_apk_path"], replace=True)
-        self.info("Setting up SD card...")
+
-        self.setup_sdcard()
+        if not self._errored:
-        self.info("Running tests...")
+            self.info("Setting up SD card...")
-        self.run_tests()
+            self.setup_sdcard(TestMode.OPTIMIZED_SHADER_COMPILATION)
+            self.info("Running optimized shader compilation tests...")
+            self.run_tests(60)
+
+        if not self._errored:
+            self.info("Setting up SD card...")
+            self.setup_sdcard(TestMode.UNOPTIMIZED_SHADER_COMPILATION)
+            self.info("Running unoptimized shader compilation tests...")
+            self.run_tests(60)
+
+        if not self._errored:
+            self.info("Setting up SD card...")
+            self.setup_sdcard(TestMode.REFTEST)
+            self.info("Running reftests...")
+            self.run_tests(60 * 30)
+
         self.info("Tests done; parsing logcat...")
         self.logcat_stop()
         self.scrape_logcat()