Bug 1486819 - support mozParams in webext search engines r=aswan,adw,mkaply

mkaply for overall search engine api changes
adw for searchservice changes.  note that a small part of it will be removed in favor of the fix from bug 1485508
aswan for webextension side.  note that I want to do better with the distribution signal, that can be in bug 1488517

Differential Revision: https://phabricator.services.mozilla.com/D4473

--HG--
extra : moz-landing-system : lando

browser/components/extensions/parent/ext-chrome-settings-overrides.js

@@ -4,6 +4,7 @@
 
 "use strict";
 
+ChromeUtils.import("resource://gre/modules/ExtensionParent.jsm");
 ChromeUtils.defineModuleGetter(this, "ExtensionPreferencesManager",
                                "resource://gre/modules/ExtensionPreferencesManager.jsm");
 ChromeUtils.defineModuleGetter(this, "ExtensionSettingsStore",
@@ -11,6 +12,10 @@ ChromeUtils.defineModuleGetter(this, "ExtensionSettingsStore",
 ChromeUtils.defineModuleGetter(this, "ExtensionControlledPopup",
                                "resource:///modules/ExtensionControlledPopup.jsm");
 
+var {
+  IconDetails,
+} = ExtensionParent;
+
 const DEFAULT_SEARCH_STORE_TYPE = "default_search";
 const DEFAULT_SEARCH_SETTING_NAME = "defaultSearch";
 const ENGINE_ADDED_SETTING_NAME = "engineAdded";
@@ -278,19 +283,31 @@ this.chrome_settings_overrides = class extends ExtensionAPI {
         Services.search.removeEngine(engines[0]);
       }
     }
+
+    let icons = extension.manifest.icons;
+    let iconURL = searchProvider.favicon_url ||
+                  (icons && extension.baseURI.resolve(IconDetails.getPreferredIcon(icons).icon));
+    let iconList = [];
+    if (icons) {
+      iconList = Object.entries(icons).map(icon => {
+        return {width: icon[0], height: icon[0],
+                url: extension.baseURI.resolve(icon[1])};
+      });
+    }
     try {
       let params = {
         template: searchProvider.search_url,
-        iconURL: searchProvider.favicon_url,
+        searchPostParams: searchProvider.search_url_post_params,
+        iconURL,
+        icons: iconList,
         alias: searchProvider.keyword,
         extensionID: extension.id,
+        isBuiltIn: extension.isPrivileged,
         suggestURL: searchProvider.suggest_url,
+        suggestPostParams: searchProvider.suggest_url_post_params,
         queryCharset: "UTF-8",
+        mozParams: searchProvider.params,
       };
-      if (searchProvider.search_url_post_params) {
-        params.method = "POST";
-        params.postData = searchProvider.search_url_post_params;
-      }
       Services.search.addEngineWithDetails(searchProvider.name.trim(), params);
       await ExtensionSettingsStore.addSetting(
         extension.id, DEFAULT_SEARCH_STORE_TYPE, ENGINE_ADDED_SETTING_NAME,

browser/components/extensions/schemas/chrome_settings_overrides.json

@@ -68,6 +68,12 @@
                     "preprocess": "localize",
                     "description": "POST parameters to the search_url as a query string."
                   },
+                  "suggest_url_post_params": {
+                    "type": "string",
+                    "optional": true,
+                    "preprocess": "localize",
+                    "description": "POST parameters to the suggest_url as a query string."
+                  },
                   "instant_url_post_params": {
                     "type": "string",
                     "optional": true,
@@ -99,6 +105,42 @@
                     "type": "boolean",
                     "optional": true,
                     "description": "Sets the default engine to a built-in engine only."
+                  },
+                  "params": {
+                    "optional": true,
+                    "type": "array",
+                    "items": {
+                      "type": "object",
+                      "properties": {
+                        "name": {
+                          "type": "string",
+                          "description": "A url parameter name"
+                        },
+                        "condition": {
+                          "type": "string",
+                          "optional": true,
+                          "enum": ["purpose", "pref"],
+                          "description": "The type of param can be either \"purpose\" or \"pref\"."
+                        },
+                        "pref": {
+                          "type": "string",
+                          "optional": true,
+                          "description": "The preference to retreive the value from."
+                        },
+                        "purpose": {
+                          "type": "string",
+                          "optional": true,
+                          "enum": ["contextmenu", "searchbar", "homepage", "keyword", "newtab"],
+                          "description": "The context that initiates a search, required if condition is \"purpose\"."
+                        },
+                        "value": {
+                          "type": "string",
+                          "optional": true,
+                          "description": "A url parameter value."
+                        }
+                      }
+                    },
+                    "description": "A list of optional search url parameters. This allows the additon of search url parameters based on how the search is performed in Firefox."
                   }
                 }
               }

browser/components/extensions/test/xpcshell/test_ext_settings_overrides_search.js

@@ -25,6 +25,10 @@ add_task(async function setup() {
 add_task(async function test_extension_adding_engine() {
   let ext1 = ExtensionTestUtils.loadExtension({
     manifest: {
+      "icons": {
+        "16": "foo.ico",
+        "32": "foo32.ico",
+      },
       "chrome_settings_overrides": {
         "search_provider": {
           "name": "MozSearch",
@@ -42,6 +46,13 @@ add_task(async function test_extension_adding_engine() {
   let engine = Services.search.getEngineByName("MozSearch");
   ok(engine, "Engine should exist.");
 
+  let {baseURI} = ext1.extension;
+  equal(engine.iconURI.spec, baseURI.resolve("foo.ico"), "icon path matches");
+  let icons = engine.getIcons();
+  equal(icons.length, 2, "both icons avialable");
+  equal(icons[0].url, baseURI.resolve("foo.ico"), "icon path matches");
+  equal(icons[1].url, baseURI.resolve("foo32.ico"), "icon path matches");
+
   let expectedSuggestURL = kSearchSuggestURL.replace("{searchTerms}", kSearchTerm);
   let submissionSuggest = engine.getSubmission(kSearchTerm, URLTYPE_SUGGEST_JSON);
   let encodedSubmissionURL = engine.getSubmission(kSearchTermIntl).uri.spec;
@@ -139,3 +150,42 @@ add_task(async function test_upgrade_default_position_engine() {
   engine = Services.search.getEngineByName("MozSearch");
   ok(!engine, "Engine should not exist");
 });
+
+add_task(async function test_extension_post_params() {
+  let ext1 = ExtensionTestUtils.loadExtension({
+    manifest: {
+      "chrome_settings_overrides": {
+        "search_provider": {
+          "name": "MozSearch",
+          "keyword": "MozSearch",
+          "search_url": kSearchEngineURL,
+          "search_url_post_params": "foo=bar&bar=foo",
+          "suggest_url": kSearchSuggestURL,
+          "suggest_url_post_params": "foo=bar&bar=foo",
+        },
+      },
+    },
+    useAddonManager: "temporary",
+  });
+
+  await ext1.startup();
+
+  let engine = Services.search.getEngineByName("MozSearch");
+  ok(engine, "Engine should exist.");
+
+  let url = engine.wrappedJSObject._getURLOfType("text/html");
+  equal(url.method, "POST", "Search URLs method is POST");
+
+  let expectedURL = kSearchEngineURL.replace("{searchTerms}", kSearchTerm);
+  let submission = engine.getSubmission(kSearchTerm);
+  equal(submission.uri.spec, expectedURL, "Search URLs should match");
+  // postData is a nsIMIMEInputStream which contains a nsIStringInputStream.
+  equal(submission.postData.data.data, "foo=bar&bar=foo", "Search postData should match");
+
+  let expectedSuggestURL = kSearchSuggestURL.replace("{searchTerms}", kSearchTerm);
+  let submissionSuggest = engine.getSubmission(kSearchTerm, URLTYPE_SUGGEST_JSON);
+  equal(submissionSuggest.uri.spec, expectedSuggestURL, "Suggest URLs should match");
+  equal(submissionSuggest.postData.data.data, "foo=bar&bar=foo", "Suggest postData should match");
+
+  await ext1.unload();
+});

browser/components/extensions/test/xpcshell/test_ext_settings_overrides_search_mozParam.js

@@ -0,0 +1,121 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+
+"use strict";
+
+ChromeUtils.import("resource://testing-common/AddonTestUtils.jsm");
+
+AddonTestUtils.init(this);
+AddonTestUtils.overrideCertDB();
+AddonTestUtils.createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "42", "42");
+
+let {
+  promiseShutdownManager,
+  promiseStartupManager,
+} = AddonTestUtils;
+
+add_task(async function setup() {
+  await promiseStartupManager();
+  Services.search.init();
+  registerCleanupFunction(async () => {
+    await promiseShutdownManager();
+  });
+});
+
+/* This tests setting moz params. */
+add_task(async function test_extension_setting_moz_params() {
+  let defaultBranch = Services.prefs.getDefaultBranch("browser.search.");
+  defaultBranch.setCharPref("param.code", "good");
+
+  // These params are conditional based on how search is initiated.
+  let mozParams = [
+    {name: "test-0", condition: "purpose", purpose: "contextmenu", value: "0"},
+    {name: "test-1", condition: "purpose", purpose: "searchbar", value: "1"},
+    {name: "test-2", condition: "purpose", purpose: "homepage", value: "2"},
+    {name: "test-3", condition: "purpose", purpose: "keyword", value: "3"},
+    {name: "test-4", condition: "purpose", purpose: "newtab", value: "4"},
+  ];
+  // These params are always included.
+  let params = [
+    {name: "simple", value: "5"},
+    {name: "term", value: "{searchTerms}"},
+    {name: "lang", value: "{language}"},
+    {name: "locale", value: "{moz:locale}"},
+    {name: "prefval", condition: "pref", pref: "code"},
+  ];
+
+  let extension = ExtensionTestUtils.loadExtension({
+    manifest: {
+      "applications": {
+        "gecko": {"id": "test@mochitest"},
+      },
+      "chrome_settings_overrides": {
+        "search_provider": {
+          "name": "MozParamsTest",
+          "search_url": "https://example.com/?q={searchTerms}",
+          "params": [...mozParams, ...params],
+        },
+      },
+    },
+    useAddonManager: "permanent",
+  });
+  await extension.startup();
+  equal(extension.extension.isPrivileged, true, "extension is priviledged");
+
+  let engine = Services.search.getEngineByName("MozParamsTest");
+
+  let extraParams = [];
+  for (let p of params) {
+    if (p.condition == "pref") {
+      extraParams.push(`${p.name}=good`);
+    } else if (p.value == "{searchTerms}") {
+      extraParams.push(`${p.name}=test`);
+    } else if (p.value == "{language}") {
+      extraParams.push(`${p.name}=${Services.locale.requestedLocale || "*"}`);
+    } else if (p.value == "{moz:locale}") {
+      extraParams.push(`${p.name}=${Services.locale.requestedLocale}`);
+    } else {
+      extraParams.push(`${p.name}=${p.value}`);
+    }
+  }
+  let paramStr = extraParams.join("&");
+
+  for (let p of mozParams) {
+    let expectedURL = engine.getSubmission("test", null, p.condition == "purpose" ? p.purpose : null).uri.spec;
+    equal(expectedURL, `https://example.com/?q=test&${p.name}=${p.value}&${paramStr}`, "search url is expected");
+  }
+
+  await extension.unload();
+});
+
+add_task(async function test_extension_setting_moz_params_fail() {
+  // Ensure that the test infra does not automatically make
+  // this privileged.
+  AddonTestUtils.usePrivilegedSignatures = false;
+
+  let extension = ExtensionTestUtils.loadExtension({
+    manifest: {
+      "applications": {
+        "gecko": {"id": "test@mochitest"},
+      },
+      "chrome_settings_overrides": {
+        "search_provider": {
+          "name": "MozParamsTest",
+          "search_url": "https://example.com/",
+          "params": [
+            {name: "testParam", condition: "purpose", purpose: "contextmenu", value: "0"},
+            {name: "prefval", condition: "pref", pref: "code"},
+            {name: "q", value: "{searchTerms}"},
+          ],
+        },
+      },
+    },
+    useAddonManager: "permanent",
+  });
+  await extension.startup();
+  equal(extension.extension.isPrivileged, false, "extension is not priviledged");
+  let engine = Services.search.getEngineByName("MozParamsTest");
+  let expectedURL = engine.getSubmission("test", null, "contextmenu").uri.spec;
+  equal(expectedURL, "https://example.com/?q=test", "engine cannot have conditional or pref params");
+  await extension.unload();
+});

browser/components/extensions/test/xpcshell/xpcshell-common.ini

@@ -9,6 +9,7 @@
 [test_ext_geckoProfiler_control.js]
 [test_ext_history.js]
 [test_ext_settings_overrides_search.js]
+[test_ext_settings_overrides_search_mozParam.js]
 [test_ext_url_overrides_newtab.js]
 [test_ext_url_overrides_newtab_update.js]
 

toolkit/components/search/nsSearchService.js

@@ -65,6 +65,7 @@ const NS_APP_USER_PROFILE_50_DIR = "ProfD";
 // We load plugins from APP_SEARCH_PREFIX, where a list.json
 // file needs to exist to list available engines.
 const APP_SEARCH_PREFIX = "resource://search-plugins/";
+const EXT_SEARCH_PREFIX = "resource://search-extensions/";
 
 // See documentation in nsIBrowserSearchService.idl.
 const SEARCH_ENGINE_TOPIC        = "browser-search-engine-modified";
@@ -1079,9 +1080,13 @@ EngineURL.prototype = {
     if (this.method == "GET") {
       // GET method requests have no post data, and append the encoded
       // query string to the url...
-      if (!url.includes("?") && dataString)
-        url += "?";
-      url += dataString;
+      if (dataString) {
+        if (url.includes("?")) {
+          url = `${url}&${dataString}`;
+        } else {
+          url = `${url}?${dataString}`;
+        }
+      }
     } else if (this.method == "POST") {
       // POST method requests must wrap the encoded text in a MIME
       // stream and supply that as POSTDATA.
@@ -1282,6 +1287,9 @@ Engine.prototype = {
   _iconUpdateURL: null,
   /* The extension ID if added by an extension. */
   _extensionID: null,
+  // If the extension is builtin we treat it as a builtin search engine as well.
+  // Both System and Distribution extensions are considered builtin for search engines.
+  _isBuiltinExtension: false,
 
   /**
    * Retrieves the data from the engine's file.
@@ -1677,6 +1685,7 @@ Engine.prototype = {
           return;
         }
         // Fall through to the data case
+      case "moz-extension":
       case "data":
         if (!this._hasPreferredIcon || aIsPreferred) {
           this._iconURI = uri;
@@ -1771,6 +1780,38 @@ Engine.prototype = {
     this._data = null;
   },
 
+  /**
+   * Initialize an EngineURL object from metadata.
+   */
+  _initEngineURLFromMetaData(aType, aParams) {
+    let url = new EngineURL(aType, aParams.method || "GET", aParams.template);
+
+    if (aParams.postParams) {
+      let queries = new URLSearchParams(aParams.postParams);
+      for (let [name, value] of queries) {
+        url.addParam(name, value);
+      }
+    }
+
+    if (aParams.mozParams) {
+      for (let p of aParams.mozParams) {
+        if ((p.condition || p.purpose) && !this._isDefault) {
+          continue;
+        }
+        if (p.condition == "pref") {
+          let value = getMozParamPref(p.pref);
+          url.addParam(p.name, value);
+          url._addMozParam(p);
+        } else {
+          url.addParam(p.name, p.value, p.purpose || undefined);
+        }
+      }
+    }
+
+    this._urls.push(url);
+    return url;
+  },
+
   /**
    * Initialize this Engine object from a collection of metadata.
    */
@@ -1779,11 +1820,24 @@ Engine.prototype = {
                 "Can't call _initFromMetaData on a readonly engine!",
                 Cr.NS_ERROR_FAILURE);
 
-    let method = aParams.method || "GET";
-    this._urls.push(new EngineURL(URLTYPE_SEARCH_HTML, method, aParams.template));
+    this._extensionID = aParams.extensionID;
+    this._isBuiltinExtension = !!aParams.isBuiltIn;
+
+    this._initEngineURLFromMetaData(URLTYPE_SEARCH_HTML, {
+      method: (aParams.searchPostParams && "POST") || aParams.method || "GET",
+      template: aParams.template,
+      postParams: aParams.searchPostParams,
+      mozParams: aParams.mozParams,
+    });
+
     if (aParams.suggestURL) {
-      this._urls.push(new EngineURL(URLTYPE_SUGGEST_JSON, "GET", aParams.suggestURL));
+      this._initEngineURLFromMetaData(URLTYPE_SUGGEST_JSON, {
+        method: (aParams.suggestPostParams && "POST") || aParams.method || "GET",
+        template: aParams.suggestURL,
+        postParams: aParams.suggestPostParams,
+      });
     }
+
     if (aParams.queryCharset) {
       this._queryCharset = aParams.queryCharset;
     }
@@ -1797,8 +1851,15 @@ Engine.prototype = {
     this._name = aName;
     this.alias = aParams.alias;
     this._description = aParams.description;
-    this._setIcon(aParams.iconURL, true);
-    this._extensionID = aParams.extensionID;
+    if (aParams.iconURL) {
+      this._setIcon(aParams.iconURL, true);
+    }
+    // Other sizes
+    if (aParams.icons) {
+      for (let icon of aParams.icons) {
+        this._addIconToMap(icon.size, icon.size, icon.url);
+      }
+    }
   },
 
   /**
@@ -2214,6 +2275,10 @@ Engine.prototype = {
   },
 
   get _isDefault() {
+    if (this._extensionID) {
+      return this._isBuiltinExtension;
+    }
+
     // If we don't have a shortName, the engine is being parsed from a
     // downloaded file, so this can't be a default engine.
     if (!this._shortName)