test optimizations and updates due to fxa changes

.github/workflows/playwright.yml

@@ -1,10 +1,10 @@
 name: Relay e2e Tests
 on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
+  schedule:
+    - cron: '0 8 * * *'
+
   workflow_dispatch:
+
 jobs:
   relaye2e:
     name: 'Relay Application e2e Tests'

.gitignore

@@ -15,3 +15,4 @@ junit.xml
 /playwright/.cache/
 state.json
 har/
+allure-results/

e2e-tests/e2eTestUtils/helpers.ts

@@ -30,41 +30,57 @@ export const deleteEmailAddressMessages = async (req: APIRequestContext, testEma
   }
 };
 
-export const checkForSignInButton = async (page: Page) => {
+const setYourPassword = async (page: Page) => {
   try {    
-    const maybeSignInButton = 'button:has-text("Sign in")'
-    await page.waitForSelector(maybeSignInButton, { timeout: 2000 })
-    await page.locator(maybeSignInButton).click()
-  } catch (error) {
-    console.error('Proceeded to logged in page')
-  }
-}
-
-export const checkForEmailInput = async (page: Page) => {
-  try {    
-    const maybeEmailInput = '.email'
-    await page.waitForSelector(maybeEmailInput, { timeout: 2000 })
-    const signInButton = page.locator('button:has-text("Sign up or sign in")')
-    await page.locator(maybeEmailInput).fill(process.env.E2E_TEST_ACCOUNT_FREE as string)
-    await signInButton.click()
     await page.locator('#password').fill(process.env.E2E_TEST_ACCOUNT_PASSWORD as string)
-    await page.locator('#submit-btn').click()
-  } catch (error) {
-    console.error('No email Proceeded to logged in page')
-  }
+    await page.locator('#vpassword').fill(process.env.E2E_TEST_ACCOUNT_PASSWORD as string)
+    await page.locator('#age').fill('31');
+    await page.locator('button:has-text("Create account")').click()
+    await checkAuthState(page)
+  } catch {}
+  
 }
 
-export const checkForVerificationCodeInput = async (page: Page) => {
-  try {    
+const enterConfirmationCode = async (page: Page) => {
+  try {        
     const maybeVerificationCodeInput = '//div[@class="card"]//input'
     await page.waitForSelector(maybeVerificationCodeInput, { timeout: 2000 })
     const confirmButton = page.locator('button:has-text("Confirm")')
     const verificationCode = await getVerificationCode(process.env.E2E_TEST_ACCOUNT_FREE as string, page)
     await page.locator(maybeVerificationCodeInput).fill(verificationCode)
     await confirmButton.click()
-  } catch (error) {
-    console.error('No email proceeding to logged in page')
-  }
+    await checkAuthState(page)
+  } catch {}
+}
+
+const signIn = async (page: Page) => {
+  try {
+    const signInButton = page.locator('#use-logged-in')
+    await Promise.all([
+      page.waitForNavigation(),
+      signInButton.click()
+    ]);
+    await checkAuthState(page)
+  } catch {}
+}
+
+const enterYourEmail = async (page: Page) => {
+  try {    
+    const maybeEmailInput = 'input[name="email"]'
+    await page.waitForSelector(maybeEmailInput, { timeout: 2000 })
+    const signInButton = page.locator('#submit-btn')
+    await page.locator(maybeEmailInput).fill(process.env.E2E_TEST_ACCOUNT_FREE as string)
+    await signInButton.click()
+    await checkAuthState(page)
+  } catch {}
+}
+
+const enterYourPassword = async (page: Page) => {
+  try {    
+    await page.locator('#password').fill(process.env.E2E_TEST_ACCOUNT_PASSWORD as string)
+    await page.locator('#submit-btn').click()
+    await checkAuthState(page)
+  } catch {}
 }
 
 export const generateRandomEmail = async () => {  
@@ -107,4 +123,33 @@ interface DefaultScreenshotOpts {
 export const defaultScreenshotOpts: Partial<DefaultScreenshotOpts> = {
   animations: 'disabled',
   maxDiffPixelRatio: 0.04
-};
+};
+
+export const checkAuthState = async (page: Page) => {
+  try {    
+    const authStateTitleString = await page.locator('h1').textContent({ timeout: 1000 })  
+    const checkIfTitleConatins = (potentialTitle: string) => {
+      return authStateTitleString?.includes(potentialTitle)
+    }
+
+    switch (true) {
+      case checkIfTitleConatins('Enter your email'):      
+        await enterYourEmail(page)
+        break;
+      case checkIfTitleConatins('Enter your password'):     
+        await enterYourPassword(page)
+        break;
+      case checkIfTitleConatins('Set your password'):      
+        await setYourPassword(page)
+        break;
+      case checkIfTitleConatins('Enter confirmation code'):      
+        await enterConfirmationCode(page)
+        break;
+      case checkIfTitleConatins('Sign in'):      
+        await signIn(page)
+        break;
+      default:
+        break;
+    }
+  } catch {}
+}

e2e-tests/pages/authPage.ts

@@ -11,7 +11,7 @@ export class AuthPage {
 
     constructor(page: Page){
         this.page = page;
-        this.emailInputField = page.locator('.email');
+        this.emailInputField = page.locator('input[name="email"]');
         this.passwordInputField = page.locator('#password');        
         this.passwordConfirmInputField = page.locator('#vpassword');
         this.ageInputField = page.locator('#age');
@@ -20,7 +20,7 @@ export class AuthPage {
     }
 
     async continue() {
-        this.continueButton.click()
+        await this.continueButton.click()
     }
 
     async enterVerificationCode(code: string){

e2e-tests/pages/dashboardPage.ts

@@ -1,5 +1,5 @@
 import { Locator, Page } from "@playwright/test";
-import { checkForEmailInput, checkForSignInButton, checkForVerificationCodeInput, getVerificationCode } from "../e2eTestUtils/helpers";
+import { checkAuthState, getVerificationCode } from "../e2eTestUtils/helpers";
 
 export class DashboardPage {
     readonly page: Page
@@ -21,7 +21,6 @@ export class DashboardPage {
     readonly relayExtensionBanner: Locator
     readonly dashBoardWithoutMasks: Locator
     readonly dashBoardWithoutMasksEmail: Locator
-    readonly generateNewMaskButton: Locator
     readonly emailsForwardedAmount: Locator
     readonly emailsBlockedAmount: Locator
     readonly emailMasksUsedAmount: Locator
@@ -68,7 +67,6 @@ export class DashboardPage {
         this.emailsForwardedAmount = page.locator('(//dd[starts-with(@class, "profile_value")])[3]')
         this.emailsBlockedAmount = page.locator('(//dd[starts-with(@class, "profile_value")])[2]')
         this.emailMasksUsedAmount = page.locator('(//dd[starts-with(@class, "profile_value")])[1]')        
-        this.generateNewMaskButton = page.locator('button:has-text("Generate new mask")')
         this.maxMaskLimitButton = page.locator('//div[starts-with(@class, "AliasList_controls")]//a[starts-with(@class, "Button_button")]')
         this.bottomUgradeBanner = page.locator('//div[starts-with(@class, "profile_bottom-banner-wrapper")]')
         this.relayExtensionBanner = page.locator('//section[starts-with(@class, "profile_banners-wrapper")]/div')
@@ -94,18 +92,21 @@ export class DashboardPage {
         this.maskCardFinalDeleteButton = page.locator('//button[contains(@class, "Button_is-destructive")]')
     } 
 
-    async open() {
+    async open() {        
         await this.page.goto('/accounts/profile/');
     }
 
     async generateMask(numberOfMasks = 1){
+        const generateNewMaskButtonString = 'button:has-text("Generate new mask")'
+        await this.page.waitForSelector(generateNewMaskButtonString, { timeout: 5000 })
+
         // check if max number of masks have been created
         if(numberOfMasks === 0){
             return
         }
            
         // generate a new mask and confirm
-        await this.generateNewMaskButton.click()
+        await this.page.locator(generateNewMaskButtonString).click()
         await this.page.waitForSelector(this.maskCard, { timeout: 3000 })
         
         // randomize between 1.5-2.5 secs between each generate to deal with issue of multiple quick clicks
@@ -186,15 +187,13 @@ export class DashboardPage {
     async sendMaskEmail(){
         // reset data
         await this.open()
-        await checkForSignInButton(this.page)
-        await checkForEmailInput(this.page)
-        await checkForVerificationCodeInput(this.page)
+        await checkAuthState(this.page)
         await this.maybeDeleteMasks()
         
         // create mask and use generated mask email to test email forwarding feature
         await this.generateMask(1)
         const generatedMaskEmail = await this.maskCardGeneratedEmail.textContent()
-    
+            
         await this.page.goto("https://monitor.firefox.com/")
     
         const checkForBreachesEmailInput = this.page.locator('#scan-email').first();

e2e-tests/specs/relay-e2e.spec.ts

@@ -1,9 +1,5 @@
 import test, { expect }  from '../fixtures/basePages'
-import {
-  checkForEmailInput,
-  checkForSignInButton,
-  checkForVerificationCodeInput,
-  deleteEmailAddressMessages } from '../e2eTestUtils/helpers';
+import { checkAuthState } from '../e2eTestUtils/helpers';
 
 test.describe.configure({ mode: 'parallel' });
 test.skip(({ browserName }) => browserName !== 'firefox', 'firefox only e2e!');
@@ -11,20 +7,18 @@ test.describe('Relay e2e function email forwarding', () => {
     // use stored authenticated state
     test.use({ storageState: 'state.json' })
 
-    test.beforeEach(async ({ dashboardPage, request }) => {      
+    test.beforeEach(async ({ dashboardPage }) => {      
       await dashboardPage.sendMaskEmail()
     });
     
-    test('Check that the user can use the masks on websites and receive emails sent to the masks, C1553068, C1553065', async ({ 
+    test('Check that the user can use the masks on websites and receive emails sent to the masks, C1553068, C1553065, C1811801', async ({ 
       dashboardPage,
       page
     }) => {
         await dashboardPage.open()
-        await checkForSignInButton(page)
-        await checkForEmailInput(page)
-        await checkForVerificationCodeInput(page)
+        await checkAuthState(page)
         const forwardedEmailCount = await dashboardPage.checkForwardedEmailCount()
-        
+                
         expect(forwardedEmailCount).toEqual('1Forwarded')        
 
         await dashboardPage.userMenuButton.click()

e2e-tests/specs/relay-general-functionality.spec.ts

@@ -1,14 +1,12 @@
 import test, { expect }  from '../fixtures/basePages'
-import { checkForEmailInput, checkForSignInButton, checkForVerificationCodeInput, defaultScreenshotOpts } from '../e2eTestUtils/helpers';
+import { checkAuthState, defaultScreenshotOpts } from '../e2eTestUtils/helpers';
 
 // using logged in state outside of describe block will cover state for all tests in file
 test.use({ storageState: 'state.json' })
 test.describe('Free - General Functionalities, Desktop', () => {
   test.beforeEach(async ({ dashboardPage, page }) => {
     await dashboardPage.open()
-    await checkForSignInButton(page)
-    await checkForEmailInput(page)
-    await checkForVerificationCodeInput(page)
+    await checkAuthState(page)
     await dashboardPage.maybeDeleteMasks()
   });
   
@@ -19,7 +17,7 @@ test.describe('Free - General Functionalities, Desktop', () => {
     expect(await dashboardPage.maxMaskLimitButton.textContent()).toContain('Get unlimited email masks')
   })
   
-  test('Check that when generating a new mask, its card is automatically opened, C1686210, C1553075', async ({ dashboardPage }) => {
+  test('Check that when generating a new mask, its card is automatically opened, C1686210, C1553075, C1553064', async ({ dashboardPage }) => {
     await dashboardPage.generateMask(1)
     
     await expect(dashboardPage.maskCardExpanded).toBeVisible()
@@ -33,9 +31,7 @@ test.describe('Free - General Functionalities, Desktop - Visual Regression', ()
 
   test.beforeEach(async ({ dashboardPage, page }) => {
     await dashboardPage.open()
-    await checkForSignInButton(page)
-    await checkForEmailInput(page)
-    await checkForVerificationCodeInput(page)
+    await checkAuthState(page)
     await dashboardPage.maybeDeleteMasks()
   });    
 

e2e-tests/specs/relay-home-page.spec.ts

@@ -47,12 +47,12 @@ test.describe('Check header buttons and their redirects,  C1812638',  () => {
     expect(page.url()).toEqual(`${process.env.E2E_TEST_BASE_URL}/`)
   }) 
 
-  test('Verify sign in button authentication flow', async ({ landingPage, authPage }) => {
+  test('Verify sign in button authentication flow, C1818784', async ({ landingPage, authPage }) => {
     await landingPage.goToSignIn()
     expect(authPage.emailInputField.isVisible()).toBeTruthy()
   })
 
-  test('Verify  sign up button authentication flow', async ({ landingPage, authPage }) => {        
+  test('Verify  sign up button authentication flow, C1818782', async ({ landingPage, authPage }) => {        
     await landingPage.goToSignUp()
     expect(authPage.emailInputField.isVisible()).toBeTruthy()
   }) 

e2e-tests/specs/relay-premium-functionality.spec.ts

@@ -1,6 +1,5 @@
 import test, { expect }  from '../fixtures/basePages'
 
-// using logged in state outside of describe block will cover state for all tests in file
 test.describe.skip('Premium - General Functionalities, Desktop', () => {
   test.beforeEach(async ({ landingPage, authPage, dashboardPage }) => {
     await landingPage.open()

e2e-tests/specs/relay-premium-upgrade.spec.ts

@@ -1,16 +1,14 @@
 import test, { expect }  from '../fixtures/basePages'
-import { checkForEmailInput, checkForSignInButton, checkForVerificationCodeInput, defaultScreenshotOpts } from '../e2eTestUtils/helpers';
+import { checkAuthState, defaultScreenshotOpts } from '../e2eTestUtils/helpers';
 
 // using logged in state outside of describe block will cover state for all tests in file
 test.use({ storageState: 'state.json' })
 test.describe.configure({ mode: 'parallel' });
 test.describe('Premium Relay - Purchase Premium Flow, Desktop', () => {
 
-  test.beforeEach(async ({ dashboardPage, page }) => {
+  test.beforeEach(async ({ dashboardPage, page, landingPage, authPage }) => {        
     await dashboardPage.open()
-    await checkForSignInButton(page)
-    await checkForEmailInput(page)
-    await checkForVerificationCodeInput(page)
+    await checkAuthState(page)
   });
 
   test('Verify that the "Upgrade" button redirects correctly,  C1812640, 1808503', async ({ dashboardPage, page }) => {
@@ -25,9 +23,7 @@ test.describe.skip(() => { // TODO: add flow for stage only
 
   test.beforeEach(async ({ dashboardPage, page }) => {
     await dashboardPage.open()
-    await checkForSignInButton(page)
-    await checkForEmailInput(page)
-    await checkForVerificationCodeInput(page)
+    await checkAuthState(page)
   });
 })
 
@@ -36,13 +32,10 @@ test.describe.skip('Premium Relay - Purchase Premium Flow, Desktop - Visual Regr
   
   test.beforeEach(async ({ dashboardPage, page }) => {
     await dashboardPage.open()
-    await checkForSignInButton(page)
-    await checkForEmailInput(page)
-    await checkForVerificationCodeInput(page)
+    await checkAuthState(page)
   });
   
-  test('Verify that the subscription page is displayed correctly, C1553108', async ({ subscriptionPage, dashboardPage, page }) => {
-    
+  test('Verify that the subscription page is displayed correctly, C1553108', async ({ subscriptionPage, dashboardPage, page }) => {    
     await dashboardPage.upgradeNow()
     expect(page.url()).toContain('subscriptions')
 

package.json

@@ -7,7 +7,8 @@
     "url": "https://github.com/mozilla/fx-private-relay/issues"
   },
   "devDependencies": {
-    "@playwright/test": "^1.24.0",
+    "@playwright/test": "^1.25.0",
+    "allure-playwright": "^2.0.0-beta.19",
     "dotenv": "^16.0.1"
   },
   "engines": {
@@ -30,8 +31,9 @@
   "scripts": {
     "test:e2e": "npx playwright test --retries=1",
     "test:headed": "npx playwright test --retries=1 --headed",
-    "test:local": "NODE_ENV=local npx playwright test --retries=2",
-    "test:stage": "NODE_ENV=stage npx playwright test --retries=2",
+    "test:report": "npx playwright show-report",
+    "test:local": "NODE_ENV=local npx playwright test --retries=2 --headed",
+    "test:stage": "NODE_ENV=stage npx playwright test --retries=2 --headed",
     "test:prod": "NODE_ENV=prod npx playwright test --retries=2 --headed",
     "heroku-prebuild": "printf '{\"commit\":\"%s\", \"commit_link\":\"https://github.com/mozilla/fx-private-relay/commit/%s\"}\n' \"$SOURCE_VERSION\" \"$SOURCE_VERSION\" > version.json",
     "heroku-postbuild": "cd frontend; NODE_ENV=\"development\" npm ci; npm run build"

playwright.config.ts

@@ -44,8 +44,9 @@ const config: PlaywrightTestConfig = {
   workers: 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: [
-    ['html']
-    // ['allure-playwright']
+    ['line'],
+    ['html'],
+    [process.env.CI ?? 'allure-playwright']
     // ['json', {  outputFile: 'test-results.json' }],
   ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */