Merge pull request #15309 from mozilla/FXA-7363

test(metrics): add functional test helpers to track metrics

.vscode/launch.json

@@ -20,7 +20,8 @@
       "type": "node",
       "protocol": "inspector",
       "env": {
-        "DEBUG": "1"
+        "DEBUG": "1",
+        "NODE_OPTIONS": "--dns-result-order=ipv4first",
       },
       "program": "${workspaceFolder}/node_modules/@playwright/test/cli.js",
       "args": ["test","--config=${workspaceFolder}/packages/functional-tests/playwright.config.ts", "--project=local"],

packages/functional-tests/lib/metrics.ts

@@ -0,0 +1,127 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { SubscribePage } from '../pages/products';
+import { expect } from '../lib/fixtures/standard';
+
+// language and time added downstream in the request handler
+// planId, productId, transformed downstream to snake_case
+// type transformed downstream to event_type
+const P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS = [
+  'flow_id',
+  // 'language',
+  'planId',
+  'productId',
+  // 'time',
+  'type',
+];
+
+// checkoutType transformed downstream to snake_case
+const P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS = ['checkoutType'];
+
+// paymentProvider, previousPlanId, previousProductId, subscriptionId transformed downstream to snake_case
+// uid transformed downstream to user_id
+const P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS = [
+  'paymentProvider',
+  'previousPlanId',
+  'previousProductId',
+  'subscriptionId',
+  'uid',
+];
+
+const REQUIRED_FIELDS_BY_EVENT_TYPE_MAP = {
+  // subscription create
+  'amplitude.subPaySetup.view': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS,
+  ],
+  'amplitude.subPaySetup.engage': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS,
+  ],
+  'amplitude.subPaySetup.submit': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS,
+    'paymentProvider',
+  ],
+  'amplitude.subPaySetup.success': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS,
+    'paymentProvider',
+    'country_code_source',
+  ],
+  'amplitude.subPaySetup.fail': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS,
+    'paymentProvider',
+    'error_id',
+  ],
+  // subscription plan change
+  'amplitude.subPaySubChange.view': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS,
+  ],
+  'amplitude.subPaySubChange.engage': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS,
+  ],
+  'amplitude.subPaySubChange.submit': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS,
+  ],
+  'amplitude.subPaySubChange.success': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS,
+  ],
+  'amplitude.subPaySubChange.fail': [
+    ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS,
+    ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS,
+    'error_id',
+  ],
+};
+
+export class MetricsObserver {
+  public rawEvents: (Object & { type: string })[];
+  private subscribePage: SubscribePage;
+
+  constructor(subscribePage: SubscribePage) {
+    this.subscribePage = subscribePage;
+    this.rawEvents = [];
+  }
+
+  startTracking() {
+    this.subscribePage.page.on('request', (request) => {
+      if (request.method() === 'POST' && request.url().includes('/metrics')) {
+        const requestBody = request.postDataJSON();
+
+        for (const eventBase of requestBody.events) {
+          const eventProperties = structuredClone(requestBody.data);
+          const rawEvent = { ...eventBase, ...eventProperties };
+
+          if (rawEvent.type in REQUIRED_FIELDS_BY_EVENT_TYPE_MAP) {
+            for (const field of REQUIRED_FIELDS_BY_EVENT_TYPE_MAP[
+              rawEvent.type
+            ]) {
+              expect(rawEvent[field]).not.toBeNull();
+              expect(rawEvent[field], `${field} is required`).toBeDefined();
+            }
+
+            // If this is an existing user checkout flow or a plan change event, uid is required
+            if (
+              rawEvent.checkoutType === 'with-account' ||
+              rawEvent.type.startsWith('amplitude.subPaySubChange')
+            ) {
+              expect(
+                rawEvent.uid,
+                'uid is required for existing accounts'
+              ).toBeDefined();
+            }
+
+            this.rawEvents.push(rawEvent);
+          }
+        }
+      }
+    });
+  }
+}

packages/functional-tests/tests/subscription-tests/subscription.spec.ts

@@ -1,10 +1,15 @@
 import { test, expect } from '../../lib/fixtures/standard';
+import { MetricsObserver } from '../../lib/metrics';
 
 test.describe.configure({ mode: 'parallel' });
 
 test.describe('subscription test with cc and paypal', () => {
-  test.beforeEach(() => {
+  let metricsObserver: MetricsObserver;
+
+  test.beforeEach(({ pages: { subscribe } }) => {
     test.slow();
+    metricsObserver = new MetricsObserver(subscribe);
+    metricsObserver.startTracking();
   });
 
   test('subscribe with credit card and login to product', async ({
@@ -48,6 +53,19 @@ test.describe('subscription test with cc and paypal', () => {
     await relier.clickEmailFirst();
     await login.submit();
     expect(await relier.isPro()).toBe(true);
+
+    const expectedEventTypes = [
+      'amplitude.subPaySetup.view',
+      'amplitude.subPaySetup.engage',
+      'amplitude.subPaySetup.submit',
+      'amplitude.subPaySetup.fail',
+      'amplitude.subPaySetup.submit',
+      'amplitude.subPaySetup.success',
+    ];
+    const actualEventTypes = metricsObserver.rawEvents.map((event) => {
+      return event.type;
+    });
+    expect(actualEventTypes).toMatchObject(expectedEventTypes);
   });
 
   test('subscribe with paypal and login to product', async ({