chore: New pipeline for v9 release (#19783)

* experiment with new pipeline

* use beachball scopes as config

* remove generate version file

* remove some steps

* use yarn instead of npm

* fix northstar scope

* update yml

* refactor(react-conformance-make-styles): react-conformance as dev deps

The extra conformance test currently takes a direct depedendency on
`@fluentui/react-conformance` to reuse types. This adds an unnecessary
coupling between v8/9 areas and makes it harder to release the libraries
separately.

Since it is very unlikely to use the package without
`react-conformance` package, moving this to dev deps

* Change files

* update md

* move react-conformance to devdeps

* fix scopes

* update current release script

* add PR comment for ts-ignore

* clean up beachball config

* remove hard coded paths

* refactor global scope var

* improve type

* add docstring

* use runPublished

* filter scopes

* PR suggestions

* add TODO

* use token in pipeline

* remove CG

* clean up runPublished

* update docstring

azure-pipelines.release-vnext.yml

@@ -0,0 +1,82 @@
+pr: none
+trigger: none
+
+# Customize build number to include major version
+# Example: v9_20201022.1
+name: 'v9_$(Date:yyyyMMdd)$(Rev:.r)'
+
+variables:
+  - group: 'Github and NPM secrets'
+  - template: .devops/templates/variables.yml
+    parameters:
+      skipComponentGovernanceDetection: false
+  - name: release.vnext # Used to scope beachball to release only vnext packages
+    value: true
+
+pool: 'Self Host Ubuntu'
+
+# TODO set schedule once the pipeline is validated after a few manual releases
+# schedules:
+#   # minute 0, hour 7 in UTC (11pm in UTC-8), Every monday
+#   # https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?tabs=yaml&view=azure-devops#supported-cron-syntax
+#   - cron: '0 7 * * 1'
+#     # will be 12am during daylight savings time unless trigger is updated
+#     displayName: 'Scheduled release (Every monday)'
+#     branches:
+#       include:
+#         - master
+
+workspace:
+  clean: all
+
+steps:
+  - template: .devops/templates/tools.yml
+
+  - script: |
+      git config user.name "Fluent UI Build"
+      git config user.email "fluentui-internal@service.microsoft.com"
+      git remote set-url origin https://$(githubUser):$(githubPAT)@github.com/microsoft/fluentui.git
+    displayName: Authenticate git for pushes
+
+  - task: Bash@3
+    inputs:
+      filePath: yarn-ci.sh
+    displayName: yarn
+
+  - script: |
+      yarn run:published build --production --no-cache
+    displayName: yarn build
+
+  - script: |
+      yarn run:published test
+    displayName: yarn test
+
+  - script: |
+      yarn run:published lint
+    displayName: yarn lint
+
+  - script: |
+      yarn publish:beachball -n $(npmToken)
+      git reset --hard origin/master
+    env:
+      GITHUB_PAT: $(githubPAT)
+    displayName: Publish changes and bump versions
+
+  # TODO update release notes script for v9
+  # - script: |
+  #     node -r ./scripts/ts-node-register ./scripts/updateReleaseNotes/index.ts --token=$(githubPAT) --apply --debug
+  #   displayName: 'Update github release notes'
+
+  # This would usually be run automatically (via a pipeline decorator from an extension), but the
+  # thorough cleanup step prevents it from working. So run it manually here.
+  - task: ComponentGovernanceComponentDetection@0
+    displayName: 'Component Detection'
+    inputs:
+      sourceScanPath: $(Agent.BuildDirectory)
+    condition: succeeded()
+    timeoutInMinutes: 5
+    continueOnError: true
+
+  - template: .devops/templates/cleanup.yml
+    parameters:
+      checkForChangedFiles: false

azure-pipelines.release.yml

@@ -44,28 +44,6 @@ steps:
       yarn generate-version-files
     displayName: Generate version files
 
-  # The contents of this file is configured in the component governence tab of ADO
-  # - task: msospo.ospo-extension.8d7f9abb-6896-461d-9e25-4f74ed65ddb2.notice@0
-  #  displayName: 'Generate NOTICE file for convergence'
-
-  # - task: DownloadBuildArtifacts@0
-  #  displayName: 'Download NOTICE.txt'
-  #  inputs:
-  #    downloadType: specific
-  #    itemPattern: '**/NOTICE.txt'
-
-  # - task: CopyFiles@2
-  #  displayName: 'Copy NOTICE to @fluentui/react-components'
-  #  inputs:
-  #    SourceFolder: '$(System.ArtifactsDirectory)'
-  #    Contents: '**/NOTICE.txt'
-  #    TargetFolder: '$(Build.SourcesDirectory)/packages/react-components'
-  #    flattenFolders: true
-
-  # - script: |
-  #    yarn copy-notices
-  #  displayName: Copy NOTICE to converged packages
-
   - script: |
       yarn run:published build --production --no-cache
     displayName: yarn build
@@ -84,7 +62,7 @@ steps:
     displayName: yarn bundle
 
   - script: |
-      npm run publish:beachball -- -b origin/master -n $(npmToken) --access public -y
+      yarn publish:beachball -n $(npmToken)
       git reset --hard origin/master
     env:
       GITHUB_PAT: $(githubPAT)

package.json

@@ -42,7 +42,7 @@
     "postinstall": "node ./scripts/postinstall.js",
     "preinstall": "node ./scripts/use-yarn-please.js",
     "prelint": "yarn satisfied && yarn syncpack:list",
-    "publish:beachball": "beachball publish --scope \"!packages/fluentui/*\"",
+    "publish:beachball": "beachball publish -b origin/master --access public -y",
     "rebuild": "node ./scripts/invalidate-just-cache.js && yarn build --reset-cache",
     "release:fluentui:canary": "node -r ./scripts/ts-node-register ./scripts/fluentui-publish publish-canary",
     "release:fluentui:minor": "node -r ./scripts/ts-node-register ./scripts/fluentui-publish publish-minor",

scripts/beachball/getScopes.ts

@@ -0,0 +1,49 @@
+import { AllPackageInfo, getAllPackageInfo, isConvergedPackage } from '../monorepo/index';
+
+/**
+ * Reads package info from the monorepo and generates the scopes for beachball bump and release.
+ * Differentiates between vNext and v8 releases.
+ *
+ * vNext scope includes all packages that have version > 8.x and shared internal packages that need versions bumped.
+ * @returns {string[]} Array of package paths for beachball scope
+ */
+export function getScopes(): string[] {
+  const allPackageInfo = getAllPackageInfo();
+
+  if (process.env.RELEASE_VNEXT) {
+    return [...getVNextPackagePaths(allPackageInfo), ...getSharedPackagePaths(allPackageInfo)];
+  }
+
+  const ignoreVNextScope = getVNextPackagePaths(allPackageInfo).map(path => `!${path}`);
+  // Northstar is never published with beachbal
+  const ignoreNorthstarScope = '!packages/fluentui/*';
+  return [ignoreNorthstarScope, ...ignoreVNextScope];
+}
+
+function getVNextPackagePaths(allPackageInfo: AllPackageInfo) {
+  return Object.values(allPackageInfo)
+    .map(packageInfo => {
+      if (isConvergedPackage(packageInfo.packageJson)) {
+        return packageInfo.packagePath;
+      }
+
+      return false;
+    })
+    .filter(Boolean) as string[];
+}
+
+function getSharedPackagePaths(allPackageInfo: AllPackageInfo) {
+  return Object.values(allPackageInfo)
+    .map(packageInfo => {
+      // These packages depend on converged packages, but are private
+      // Can be included in the publish scope so that dependencies are bumped correctly.
+      const privateNonConverged = ['perf-test', 'vr-tests'];
+
+      if (privateNonConverged.includes(packageInfo.packageJson.name)) {
+        return packageInfo.packagePath;
+      }
+
+      return false;
+    })
+    .filter(Boolean) as string[];
+}

scripts/beachball/getVNextChangelogGroups.ts

@@ -0,0 +1,37 @@
+import { ChangelogGroupOptions } from 'beachball';
+import { AllPackageInfo, getAllPackageInfo } from '../monorepo/index';
+
+/**
+ * Generates grouped changelog for vNext
+ * @returns Grouped changelog configuration
+ */
+export function getVNextChangelogGroups(): ChangelogGroupOptions {
+  const allPackageInfo = getAllPackageInfo();
+  const vNextPackagePaths = getVNextPackagePaths(allPackageInfo);
+
+  return {
+    masterPackageName: '@fluentui/react-components',
+    changelogPath: 'packages/react-components',
+    include: vNextPackagePaths,
+  };
+}
+
+export function getVNextPackagePaths(allPackageInfo: AllPackageInfo) {
+  return Object.values(allPackageInfo)
+    .map(packageInfo => {
+      if (packageInfo.packageJson.version.startsWith('9.')) {
+        return packageInfo.packagePath;
+      }
+
+      // These packages depend on converged packages, but are private
+      // Can be included in the publish scope so that dependencies are bumped correctly.
+      const privateNonConverged = ['perf-test', 'vr-tests'];
+
+      if (privateNonConverged.includes(packageInfo.packageJson.name)) {
+        return packageInfo.packagePath;
+      }
+
+      return false;
+    })
+    .filter(Boolean) as string[];
+}

scripts/beachball/index.ts

@@ -1,33 +1,18 @@
 import { BeachballConfig } from 'beachball';
-import { getAllPackageInfo, isConvergedPackage } from '../monorepo/index';
 import { renderHeader, renderEntry } from './customRenderers';
-
-const allPackageInfo = getAllPackageInfo();
-const fluentConvergedPackagePaths = Object.values(allPackageInfo)
-  .map(packageInfo => {
-    if (isConvergedPackage(packageInfo.packageJson) && !packageInfo.packageJson.private) {
-      return packageInfo.packagePath;
-    }
-
-    return false;
-  })
-  .filter(Boolean) as string[];
+import { getScopes } from './getScopes';
+import { getVNextChangelogGroups } from './getVNextChangelogGroups';
 
 export const config: BeachballConfig = {
   disallowedChangeTypes: ['major', 'prerelease'],
   tag: 'latest',
   generateChangelog: true,
+  scope: getScopes(),
   changelog: {
     customRenderers: {
       renderHeader,
       renderEntry,
     },
-    groups: [
-      {
-        masterPackageName: '@fluentui/react-components',
-        changelogPath: 'packages/react-components',
-        include: fluentConvergedPackagePaths,
-      },
-    ],
+    groups: [getVNextChangelogGroups()],
   },
 };

scripts/monorepo/isConvergedPackage.js

@@ -2,17 +2,19 @@
 const semver = require('semver');
 const { readConfig } = require('../read-config');
 
+/**
+ *  @typedef {string | import('./index').PackageJson}  PathOrPackageJson
+ */
+
 /**
  * Determines whether a package is converged, based on its version.
- * @param {string} [packagePathOrJson] optional different package path to run in OR previously-read package.json
+ * @param {PathOrPackageJson} [packagePathOrJson] optional different package path to run in OR previously-read package.json
  * (defaults to reading package.json from `process.cwd()`)
  * @returns {boolean} true if it's a converged package (version >= 9)
  */
 function isConvergedPackage(packagePathOrJson) {
   const packageJson =
-    !packagePathOrJson || typeof packagePathOrJson === 'string'
-      ? readConfig('package.json', packagePathOrJson)
-      : packagePathOrJson;
+    typeof packagePathOrJson === 'string' ? readConfig('package.json', packagePathOrJson) : packagePathOrJson;
   return !!packageJson && semver.major(packageJson.version) >= 9;
 }
 

scripts/monorepo/runPublished.js

@@ -1,6 +1,7 @@
 // @ts-check
 const { spawnSync } = require('child_process');
 const getAllPackageInfo = require('./getAllPackageInfo');
+const isConvergedPackage = require('./isConvergedPackage');
 const lageBin = require.resolve('lage/bin/lage.js');
 
 const argv = process.argv.slice(2);
@@ -27,12 +28,20 @@ const websitePackages = [
 // (which must be built and uploaded with each release). This is similar to "--scope \"!packages/fluentui/*\""
 // in the root package.json's publishing-related scripts and will need to be updated if --scope changes.
 const beachballPackageScopes = Object.entries(getAllPackageInfo())
-  .filter(
-    ([, { packageJson, packagePath }]) =>
-      !/[\\/]fluentui[\\/]/.test(packagePath) &&
-      (packageJson.private !== true || websitePackages.includes(packageJson.name)),
-  )
-  .map(([packageName]) => `--to=${packageName}`);
+  .filter(([, { packageJson, packagePath }]) => {
+    // Ignore northstar and private packages
+    if (/[\\/]fluentui[\\/]/.test(packagePath) || packageJson.private === true) {
+      return false;
+    }
+
+    if (process.env.RELEASE_VNEXT) {
+      return isConvergedPackage(packageJson);
+    }
+
+    return websitePackages.includes(packageJson.name);
+  })
+  .map(([packageName]) => `--to=${packageName}`)
+  .filter(Boolean);
 
 const lageArgs = [
   'run',