From c00009a3f3316c5233c620dbcc6eb0bfd0eee814 Mon Sep 17 00:00:00 2001 From: Meghan Sardesai Date: Tue, 16 May 2023 10:00:55 -0400 Subject: [PATCH] test(metrics): add functional test helpers to track metrics Because: * We have little in the way of metrics tests in our functional tests currently. This commit: * Adds test helpers to assert our events are being recorded as expected. Closes FXA-7363 --- .vscode/launch.json | 3 +- packages/functional-tests/lib/metrics.ts | 127 ++++++++++++++++++ .../subscription-tests/subscription.spec.ts | 20 ++- 3 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 packages/functional-tests/lib/metrics.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 2befc86799..74d8c86583 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,7 +20,8 @@ "type": "node", "protocol": "inspector", "env": { - "DEBUG": "1" + "DEBUG": "1", + "NODE_OPTIONS": "--dns-result-order=ipv4first", }, "program": "${workspaceFolder}/node_modules/@playwright/test/cli.js", "args": ["test","--config=${workspaceFolder}/packages/functional-tests/playwright.config.ts", "--project=local"], diff --git a/packages/functional-tests/lib/metrics.ts b/packages/functional-tests/lib/metrics.ts new file mode 100644 index 0000000000..860bf3d622 --- /dev/null +++ b/packages/functional-tests/lib/metrics.ts @@ -0,0 +1,127 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { SubscribePage } from '../pages/products'; +import { expect } from '../lib/fixtures/standard'; + +// language and time added downstream in the request handler +// planId, productId, transformed downstream to snake_case +// type transformed downstream to event_type +const P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS = [ + 'flow_id', + // 'language', + 'planId', + 'productId', + // 'time', + 'type', +]; + +// checkoutType transformed downstream to snake_case +const P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS = ['checkoutType']; + +// paymentProvider, previousPlanId, previousProductId, subscriptionId transformed downstream to snake_case +// uid transformed downstream to user_id +const P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS = [ + 'paymentProvider', + 'previousPlanId', + 'previousProductId', + 'subscriptionId', + 'uid', +]; + +const REQUIRED_FIELDS_BY_EVENT_TYPE_MAP = { + // subscription create + 'amplitude.subPaySetup.view': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS, + ], + 'amplitude.subPaySetup.engage': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS, + ], + 'amplitude.subPaySetup.submit': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS, + 'paymentProvider', + ], + 'amplitude.subPaySetup.success': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS, + 'paymentProvider', + 'country_code_source', + ], + 'amplitude.subPaySetup.fail': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_CREATE_REQUIRED_FIELDS, + 'paymentProvider', + 'error_id', + ], + // subscription plan change + 'amplitude.subPaySubChange.view': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS, + ], + 'amplitude.subPaySubChange.engage': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS, + ], + 'amplitude.subPaySubChange.submit': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS, + ], + 'amplitude.subPaySubChange.success': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS, + ], + 'amplitude.subPaySubChange.fail': [ + ...P1_PAYMENTS_GLOBAL_REQUIRED_FIELDS, + ...P1_SUBSCRIPTION_PLAN_CHANGE_REQUIRED_FIELDS, + 'error_id', + ], +}; + +export class MetricsObserver { + public rawEvents: (Object & { type: string })[]; + private subscribePage: SubscribePage; + + constructor(subscribePage: SubscribePage) { + this.subscribePage = subscribePage; + this.rawEvents = []; + } + + startTracking() { + this.subscribePage.page.on('request', (request) => { + if (request.method() === 'POST' && request.url().includes('/metrics')) { + const requestBody = request.postDataJSON(); + + for (const eventBase of requestBody.events) { + const eventProperties = structuredClone(requestBody.data); + const rawEvent = { ...eventBase, ...eventProperties }; + + if (rawEvent.type in REQUIRED_FIELDS_BY_EVENT_TYPE_MAP) { + for (const field of REQUIRED_FIELDS_BY_EVENT_TYPE_MAP[ + rawEvent.type + ]) { + expect(rawEvent[field]).not.toBeNull(); + expect(rawEvent[field], `${field} is required`).toBeDefined(); + } + + // If this is an existing user checkout flow or a plan change event, uid is required + if ( + rawEvent.checkoutType === 'with-account' || + rawEvent.type.startsWith('amplitude.subPaySubChange') + ) { + expect( + rawEvent.uid, + 'uid is required for existing accounts' + ).toBeDefined(); + } + + this.rawEvents.push(rawEvent); + } + } + } + }); + } +} diff --git a/packages/functional-tests/tests/subscription-tests/subscription.spec.ts b/packages/functional-tests/tests/subscription-tests/subscription.spec.ts index 89bcfe5b7a..25a528a29d 100644 --- a/packages/functional-tests/tests/subscription-tests/subscription.spec.ts +++ b/packages/functional-tests/tests/subscription-tests/subscription.spec.ts @@ -1,10 +1,15 @@ import { test, expect } from '../../lib/fixtures/standard'; +import { MetricsObserver } from '../../lib/metrics'; test.describe.configure({ mode: 'parallel' }); test.describe('subscription test with cc and paypal', () => { - test.beforeEach(() => { + let metricsObserver: MetricsObserver; + + test.beforeEach(({ pages: { subscribe } }) => { test.slow(); + metricsObserver = new MetricsObserver(subscribe); + metricsObserver.startTracking(); }); test('subscribe with credit card and login to product', async ({ @@ -48,6 +53,19 @@ test.describe('subscription test with cc and paypal', () => { await relier.clickEmailFirst(); await login.submit(); expect(await relier.isPro()).toBe(true); + + const expectedEventTypes = [ + 'amplitude.subPaySetup.view', + 'amplitude.subPaySetup.engage', + 'amplitude.subPaySetup.submit', + 'amplitude.subPaySetup.fail', + 'amplitude.subPaySetup.submit', + 'amplitude.subPaySetup.success', + ]; + const actualEventTypes = metricsObserver.rawEvents.map((event) => { + return event.type; + }); + expect(actualEventTypes).toMatchObject(expectedEventTypes); }); test('subscribe with paypal and login to product', async ({