Merge pull request #14542 from mozilla/run-subscription-tests-on-CI

Increase resource size and fix a few flaky tests and enable subplat tests to run on CircleCi

.circleci/config.yml

@@ -308,7 +308,7 @@ jobs:
           path: artifacts/tests
 
   playwright-functional-tests:
-    resource_class: large
+    resource_class: xlarge
     parallelism: 5
     docker:
       - image: mcr.microsoft.com/playwright:v1.28.0-focal
@@ -321,6 +321,8 @@ jobs:
       NODE_ENV: dev
       TRACING_SERVICE_NAME: ci-playwright-functional-tests
       TRACING_CONSOLE_EXPORTER_ENABLED: true
+      SUBSCRIPTIONS_PAYPAL_ENABLED: true
+      PAYPAL_SANDBOX: true
     steps:
       # needed by check-mysql.sh
       - run: apt-get update && apt-get install -y netcat
@@ -328,6 +330,11 @@ jobs:
       - run:
           name: Install playwright
           command: npx playwright install --with-deps
+      - run:
+          name: Add localhost
+          command: |
+            tee -a /etc/hosts \<<<'127.0.0.1 localhost'
+            cat /etc/hosts
       - run:
           name: Running playwright tests
           command: ./packages/functional-tests/scripts/test-ci.sh

packages/functional-tests/pages/products/index.ts

@@ -2,7 +2,9 @@ import { BaseLayout } from '../layout';
 
 export class SubscribePage extends BaseLayout {
   setFullName(name: string = 'Cave Johnson') {
-    return this.page.fill('[data-testid="name"]', name);
+    const input = this.page.locator('[data-testid="name"]');
+    input.waitFor({ state: 'attached' });
+    return input.fill(name);
   }
 
   async setCreditCardInfo() {
@@ -25,16 +27,25 @@ export class SubscribePage extends BaseLayout {
   }
 
   async clickPayNow() {
-    await this.page.click('button[type=submit]');
+    const pay = this.page.locator('[data-testid="submit"]');
+    await pay.waitFor();
+    await pay.click();
   }
 
   async setPayPalInfo() {
     await this.page.check('[data-testid="confirm"]');
+    const paypalButton = this.page.locator(
+      '[data-testid="paypal-button-container"]'
+    );
+    await paypalButton.waitFor({ state: 'attached' });
     const [paypalWindow] = await Promise.all([
       this.page.waitForEvent('popup'),
       this.page.click('[data-testid="paypal-button-container"]'),
     ]);
     await paypalWindow.waitForLoadState('load');
+    await paypalWindow.waitForNavigation({
+      url: /checkoutnow/,
+    });
     await paypalWindow.fill(
       'input[type=email]',
       'qa-test-no-balance-16@personal.example.com'
@@ -47,10 +58,10 @@ export class SubscribePage extends BaseLayout {
   }
 
   async addCouponCode(code) {
-    this.page.click('[data-testid="coupon-input"]');
-    this.page.fill('[data-testid="coupon-input"]', code);
+    const input = this.page.locator('[data-testid="coupon-input"]');
+    await input.waitFor({ state: 'attached' });
+    await input.fill(code);
     this.page.click('[data-testid="coupon-button"]');
-
   }
 
   async clickTryAgain() {
@@ -66,30 +77,37 @@ export class SubscribePage extends BaseLayout {
   }
 
   async discountAppliedSuccess() {
-    const discount = this.page.locator('.coupon-header:has-text("Discount Reward Applied")');
+    const discount = this.page.locator(
+      '[data-testid="coupon-component"]:has-text("Promo Code Applied")'
+    );
     await discount.waitFor();
     return discount.isVisible();
   }
 
   async oneTimeDiscountSuccess() {
-    const discount = this.page.locator
-      ('[data-testid="coupon-success"]:has-text("Your plan will automatically renew at the list price.")');
+    const discount = this.page.locator(
+      '[data-testid="coupon-success"]:has-text("Your plan will automatically renew at the list price.")'
+    );
     await discount.waitFor();
     return discount.isVisible();
   }
 
   async discountListPrice() {
-    const listPrice = this.page.locator('.plan-details-total:has-text("List Price")');
+    const listPrice = this.page.locator(
+      '.plan-details-item:has-text("List Price")'
+    );
     return listPrice.isVisible();
   }
 
   async discountLineItem() {
-    const disc = this.page.locator('.plan-details-total:has-text("Discount")');
+    const disc = this.page.locator('.plan-details-item:has-text("Promo Code")');
     return disc.isVisible();
   }
 
   async discountTextbox() {
-    const discount = this.page.locator('.coupon-header:has-text("Discount")');
+    const discount = this.page.locator(
+      '.coupon-component:has-text("Promo Code")'
+    );
     await discount.waitFor();
     return discount.isVisible();
   }

packages/functional-tests/pages/relier.ts

@@ -10,12 +10,14 @@ export class RelierPage extends BaseLayout {
 
   async isLoggedIn() {
     const login = this.page.locator('#loggedin');
-    await login.waitFor();
+    await login.waitFor({ state: 'visible' });
     return login.isVisible();
   }
 
-  isPro() {
-    return this.page.isVisible('.pro-status', { timeout: 1000 });
+  async isPro() {
+    const pro = this.page.locator('.pro-status');
+    await pro.waitFor({ state: 'visible' });
+    return pro.isVisible();
   }
 
   async signOut() {

packages/functional-tests/pages/settings/layout.ts

@@ -62,7 +62,7 @@ export abstract class SettingsLayout extends BaseLayout {
     await this.clickAvatarIcon();
     await Promise.all([
       this.clickSignOut(),
-      this.page.waitForURL(this.target.baseUrl, { waitUntil: 'networkidle' }),
+      this.page.waitForURL(this.target.baseUrl, { waitUntil: 'load' }),
     ]);
   }
 }

packages/functional-tests/playwright.config.ts

@@ -14,7 +14,7 @@ const SLOWMO = parseInt(process.env.PLAYWRIGHT_SLOWMO || '0');
 const config: PlaywrightTestConfig<TestOptions, WorkerOptions> = {
   outputDir: path.resolve(__dirname, '../../artifacts/functional'),
   forbidOnly: CI,
-  retries: CI ? 1 : 0,
+  retries: CI ? 2 : 0,
   testDir: 'tests',
   use: {
     viewport: { width: 1280, height: 720 },
@@ -67,7 +67,7 @@ const config: PlaywrightTestConfig<TestOptions, WorkerOptions> = {
       ]
     : 'list',
   workers: CI ? 2 : undefined,
-  maxFailures: CI ? 2 : 0,
+  maxFailures: CI ? 3 : 0,
 };
 
 export default config;

packages/functional-tests/tests/settings/recoveryKey.spec.ts

@@ -167,7 +167,7 @@ test.describe('recovery key test', () => {
       EmailType.recovery,
       EmailHeader.link
     );
-    await page.goto(link2, { waitUntil: 'networkidle' });
+    await page.goto(link2, { waitUntil: 'load' });
 
     // Verify reset link expired
     expect(await login.resetPasswordLinkExpriredHeader()).toBe(true);

packages/functional-tests/tests/settings/totp.spec.ts

@@ -140,7 +140,7 @@ test.describe('two step auth', () => {
       EmailType.lowRecoveryCodes,
       EmailHeader.link
     );
-    await page.goto(link, { waitUntil: 'networkidle' });
+    await page.goto(link, { waitUntil: 'load' });
     const newCodes = await totp.getRecoveryCodes();
     expect(newCodes.length).toEqual(recoveryCodes.length);
   });

packages/functional-tests/tests/subscription-tests/coupons-ui-tests/coupon.spec.ts

@@ -1,14 +1,11 @@
 import { test, expect } from '../../../lib/fixtures/standard';
 
 test.describe('coupon test', () => {
-  test.beforeEach(({ }, { project }) => {
+  test.beforeEach(() => {
     test.slow();
-    test.skip(project.name !== 'stage', "Only run these tests in 'stage' env");
   });
 
-  test('apply an expired coupon', async ({
-    pages: { relier, subscribe },
-  }) => {
+  test('apply an expired coupon', async ({ pages: { relier, subscribe } }) => {
     await relier.goto();
     await relier.clickSubscribe6Month();
 
@@ -21,9 +18,7 @@ test.describe('coupon test', () => {
     );
   });
 
-  test('apply an invalid coupon', async ({
-    pages: { relier, subscribe },
-  }) => {
+  test('apply an invalid coupon', async ({ pages: { relier, subscribe } }) => {
     await relier.goto();
     await relier.clickSubscribe6Month();
 
@@ -148,7 +143,7 @@ test.describe('coupon test', () => {
 
   test('subscribe with paypal and use coupon', async ({
     pages: { relier, login, subscribe },
-  }, ) => {
+  }) => {
     await relier.goto();
     await relier.clickSubscribe12Month();
 
@@ -183,7 +178,6 @@ test.describe('coupon test', () => {
     await subscribe.removeCouponCode();
 
     // Verify the discount is removed
-    expect(await subscribe.discountListPrice()).toBe(false);
     expect(await subscribe.discountLineItem()).toBe(false);
   });
 });

packages/functional-tests/tests/subscription-tests/coupons-ui-tests/ui-tests.spec.ts

@@ -1,14 +1,13 @@
 import { test, expect } from '../../../lib/fixtures/standard';
 
 test.describe('ui functionality', () => {
-  test.beforeEach(({ }, { project }) => {
-    test.skip(project.name !== 'stage', "Only run these tests in 'stage' env");
+  test.beforeEach(() => {
+    test.slow();
   });
 
   test('verify discount textbox display', async ({
     pages: { relier, subscribe },
   }) => {
-    test.slow();
     await relier.goto();
     await relier.clickSubscribe6Month();
 

packages/functional-tests/tests/subscription-tests/subscription.spec.ts

@@ -1,14 +1,13 @@
 import { test, expect } from '../../lib/fixtures/standard';
 
 test.describe('subscription test with cc and paypal', () => {
-  test.beforeEach(({}, { project }) => {
-    test.skip(project.name !== 'stage', "Only run these tests in 'stage' env");
+  test.beforeEach(() => {
+    test.slow();
   });
 
   test('subscribe with credit card and login to product', async ({
     pages: { relier, login, subscribe },
   }) => {
-    test.slow();
     await relier.goto();
     await relier.clickSubscribe();
     await subscribe.setFullName();
@@ -24,7 +23,6 @@ test.describe('subscription test with cc and paypal', () => {
   test('subscribe with credit card after initial failed subscription', async ({
     pages: { relier, login, subscribe },
   }) => {
-    test.slow();
     await relier.goto();
     await relier.clickSubscribe();
     await subscribe.setFullName();
@@ -42,8 +40,7 @@ test.describe('subscription test with cc and paypal', () => {
 
   test('subscribe with paypal and login to product', async ({
     pages: { relier, login, subscribe },
-  }, { project }) => {
-    test.slow();
+  }) => {
     await relier.goto();
     await relier.clickSubscribe();
     await subscribe.setPayPalInfo();