Bug 1752178 - Add organic:none field for search telemetry. r=Standard8

This patch adds a new field, expectedOrganicCodes to search-telemetry-v2. It
is an array of strictly matched partner codes.

Searches can be reported as organic:none for Telemetry when the search is
truely organic, when the user searches by visiting the DuckDuckGo's website
rather than through one of our Search Access Points.

Differential Revision: https://phabricator.services.mozilla.com/D140969

browser/components/search/SearchSERPTelemetry.jsm

@@ -491,6 +491,8 @@ class TelemetryHandler {
         } else if (searchProviderInfo.organicCodes.includes(code)) {
           oldType = "organic";
           type = "organic";
+        } else if (searchProviderInfo.expectedOrganicCodes.includes(code)) {
+          code = "none";
         } else {
           code = "other";
         }

browser/components/search/schema/search-telemetry-schema.json

@@ -36,10 +36,19 @@
         "pattern": "^[a-zA-Z0-9-._]*$"
       }
     },
+    "expectedOrganicCodes": {
+      "type": "array",
+      "title": "Expected Organic Codes",
+      "description": "An array of partner codes to match against the parameters in the url. Matching these codes will report the SERP as organic:none which means the user has done a search through the search engine's website rather than through SAP.",
+      "items": {
+        "type": "string",
+        "pattern": "^[a-zA-Z0-9-._]*$"
+      }
+    },
     "organicCodes": {
       "type": "array",
       "title": "Organic Codes",
-      "description": "An array of partner codes to match against the parameters in the url.  Matching these codes will report the SERP as organic.",
+      "description": "An array of partner codes to match against the parameters in the url. Matching these codes will report the SERP as organic:<partner code>, which means the search was performed organically rather than through a SAP.",
       "items": {
         "type": "string",
         "pattern": "^[a-zA-Z0-9-._]*$"

browser/components/search/schema/search-telemetry-ui-schema.json

@@ -5,6 +5,7 @@
     "queryParamName",
     "codeParamName",
     "taggedCodes",
+    "expectedOrganicCodes",
     "organicCodes",
     "followOnParamNames",
     "followOnCookies",

browser/components/search/test/unit/test_urlTelemetry.js

@@ -160,6 +160,22 @@ const TESTS = [
     adUrls: ["https://duckduckgo.com/y.js?ad_provider=foo"],
     nonAdUrls: ["https://duckduckgo.com/?q=foo&t=ffab&ia=images&iax=images"],
   },
+  {
+    title: "DuckDuckGo expected organic code",
+    trackingUrl: "https://duckduckgo.com/?q=test&t=h_&ia=news",
+    expectedSearchCountEntry: "duckduckgo.in-content:organic:none",
+    expectedAdKey: "duckduckgo:organic",
+    adUrls: ["https://duckduckgo.com/y.js?ad_provider=foo"],
+    nonAdUrls: ["https://duckduckgo.com/?q=foo&t=ffab&ia=images&iax=images"],
+  },
+  {
+    title: "DuckDuckGo expected organic code 2",
+    trackingUrl: "https://duckduckgo.com/?q=test&t=hz&ia=news",
+    expectedSearchCountEntry: "duckduckgo.in-content:organic:none",
+    expectedAdKey: "duckduckgo:organic",
+    adUrls: ["https://duckduckgo.com/y.js?ad_provider=foo"],
+    nonAdUrls: ["https://duckduckgo.com/?q=foo&t=ffab&ia=images&iax=images"],
+  },
   {
     title: "DuckDuckGo organic no code",
     trackingUrl: "https://duckduckgo.com/?q=test&ia=news",

browser/components/search/test/unit/test_urlTelemetry_generic.js

@@ -22,6 +22,7 @@ const TEST_PROVIDER_INFO = [
     queryParamName: "q",
     codeParamName: "abc",
     taggedCodes: ["ff", "tb"],
+    expectedOrganicCodes: ["baz"],
     organicCodes: ["foo"],
     followOnParamNames: ["a"],
     extraAdServersRegexps: [/^https:\/\/www\.example\.com\/ad2/],
@@ -69,6 +70,14 @@ const TESTS = [
     adUrls: ["https://www.example.com/ad2"],
     nonAdUrls: ["https://www.example.com/ad3"],
   },
+  {
+    title: "Organic search expected organic matched code",
+    trackingUrl: "https://www.example.com/search?q=test&abc=baz",
+    expectedSearchCountEntry: "example.in-content:organic:none",
+    expectedAdKey: "example:organic",
+    adUrls: ["https://www.example.com/ad2"],
+    nonAdUrls: ["https://www.example.com/ad3"],
+  },
   {
     title: "Organic search no codes",
     trackingUrl: "https://www.example.com/search?q=test",

services/settings/dumps/main/search-telemetry-v2.json

@@ -21,6 +21,7 @@
       ],
       "telemetryId": "bing",
       "organicCodes": [],
+      "expectedOrganicCodes": [],
       "codeParamName": "pc",
       "queryParamName": "q",
       "followOnCookies": [
@@ -71,6 +72,7 @@
       ],
       "telemetryId": "google",
       "organicCodes": [],
+      "expectedOrganicCodes": [],
       "codeParamName": "client",
       "queryParamName": "q",
       "searchPageRegexp": "^https://www\\.google\\.(?:.+)/search",
@@ -105,6 +107,7 @@
       ],
       "telemetryId": "duckduckgo",
       "organicCodes": [],
+      "expectedOrganicCodes": ["hz", "h_"],
       "codeParamName": "t",
       "queryParamName": "q",
       "searchPageRegexp": "^https://duckduckgo\\.com/",
@@ -133,6 +136,7 @@
       ],
       "telemetryId": "baidu",
       "organicCodes": [],
+      "expectedOrganicCodes": [],
       "codeParamName": "tn",
       "queryParamName": "wd",
       "searchPageRegexp": "^https://www\\.baidu\\.com/(?:s|baidu)",