Bug 1597963 - Pass VT_ERROR for Explorer to call ShellExecuteExW with null verb. r=aklotz

The patch for Bug 1588975 specified the "open" verb to execute a target, but
the default verb is not always "open".  For example, the default verb for a font
file is "preview".  We should specify null verb to start the default operation.

Now we use `IShellDispatch2.ShellExecute` to ask explorer.exe to call
`ShellExecuteExW`.  That method takes an optional `VARIANT` parameter as a verb.
According to https://devblogs.microsoft.com/oldnewthing/20140919-00/?p=44023,
we need to pass `VT_ERROR` to omit an optional parameter.  If we pass
other values such as `nullptr` with `VT_BSTR` or `VT_EMPTY`, explorer.exe calls
`ShellExecuteExW` with the empty string `""` instead of `nullptr`, which is not
considered as a valid verb if the target file is not associated with any app.

Differential Revision: https://phabricator.services.mozilla.com/D54036

--HG--
extra : moz-landing-system : lando

uriloader/exthandler/win/nsMIMEInfoWin.cpp

@@ -55,7 +55,9 @@ static nsresult ShellExecuteWithIFile(const nsCOMPtr<nsIFile>& aExecutable,
   }
 
   _bstr_t execPathBStr(execPath.get());
-  _variant_t verb(L"open");
+  // Pass VT_ERROR/DISP_E_PARAMNOTFOUND to omit an optional RPC parameter
+  // to execute a file with the default verb.
+  _variant_t verbDefault(DISP_E_PARAMNOTFOUND, VT_ERROR);
   _variant_t workingDir;
   _variant_t showCmd(SW_SHOWNORMAL);
 
@@ -63,7 +65,7 @@ static nsresult ShellExecuteWithIFile(const nsCOMPtr<nsIFile>& aExecutable,
   // Skype for Business do not start correctly when inheriting our process's
   // migitation policies.
   mozilla::LauncherVoidResult shellExecuteOk = mozilla::ShellExecuteByExplorer(
-      execPathBStr, aArgs, verb, workingDir, showCmd);
+      execPathBStr, aArgs, verbDefault, workingDir, showCmd);
   if (shellExecuteOk.isErr()) {
     return NS_ERROR_FILE_EXECUTION_FAILED;
   }

xpcom/io/nsLocalFileWin.cpp

@@ -3021,7 +3021,9 @@ nsLocalFile::Launch() {
   _bstr_t execPath(mResolvedPath.get());
 
   _variant_t args;
-  _variant_t verb(L"open");
+  // Pass VT_ERROR/DISP_E_PARAMNOTFOUND to omit an optional RPC parameter
+  // to execute a file with the default verb.
+  _variant_t verbDefault(DISP_E_PARAMNOTFOUND, VT_ERROR);
   _variant_t workingDir;
   _variant_t showCmd(SW_SHOWNORMAL);
 
@@ -3040,7 +3042,7 @@ nsLocalFile::Launch() {
   // Skype for Business do not start correctly when inheriting our process's
   // migitation policies.
   mozilla::LauncherVoidResult shellExecuteOk = mozilla::ShellExecuteByExplorer(
-      execPath, args, verb, workingDir, showCmd);
+      execPath, args, verbDefault, workingDir, showCmd);
   if (shellExecuteOk.isErr()) {
     return NS_ERROR_FILE_EXECUTION_FAILED;
   }