build: Remove lerna from client and build-tools (#16161)

We currently use lerna in some places in CI, but it is called via npx
and thus its depednencies are not locked to a particular version., This
means that lerna releases can break our pipelines. This PR removes lerna
from the client and build-tools pipelines, as well as the independent
package pipelines. Server release groups are left alone.

- Removes all lerna dependencies from the client and build-tools release
groups.
- `lerna ls --toposort` has been replaced by a new command, `flub list`.
The output of this command does not exactly match lerna's but it still
seems to be sorted appropriately.
- `lerna exec` uses in the pipeline have been replaced by `flub exec`.
- Adds a .releaseGroup file to the root of each release group. I
couldn't think of a simpler way to make the pipeline behave differently
for release groups vs. independent packages. This is needed because the
pack step in the pipeline uses the presence of the file to determine
whether it needs to pack multiple packages or not.

### Not included in this PR

Server release groups still use lerna. Changing them will be more
difficult because they build in a docker context that is rooted at the
release gropup root. I need to figure out a solution for them, so I'll
do that separately from this PR.

---------

Co-authored-by: Alex Villarreal <716334+alexvy86@users.noreply.github.com>
Co-authored-by: Craig Macomber (Microsoft) <42876482+CraigMacomber@users.noreply.github.com>

.releaseGroup

@@ -0,0 +1 @@
+The presence of this file in a directory indicates it is the root of a release group.

build-tools/.releaseGroup

@@ -0,0 +1 @@
+The presence of this file in a directory indicates it is the root of a release group.

build-tools/package.json

@@ -12,25 +12,23 @@
 	"author": "Microsoft and contributors",
 	"scripts": {
 		"build": "npm run build:compile && npm run lint && npm run build:docs",
-		"build:ci": "lerna run build:compile --stream",
-		"build:compile": "lerna run build:compile --stream",
-		"build:docs": "lerna run build:docs --stream --parallel",
+		"build:compile": "pnpm run -r --stream build:compile",
+		"build:docs": "pnpm run -r --no-sort --stream build:docs",
 		"build:fast": "fluid-build --root .. -g build-tools",
 		"changelog": "conventional-changelog -p conventionalcommits -i CHANGELOG.md -s -r 2 --commit-path . -t build-tools_v --context lerna.json",
-		"ci:build": "lerna run build:compile --stream",
-		"ci:build:docs": "lerna run ci:build:docs --stream --parallel",
-		"ci:eslint": "lerna run eslint --no-sort --stream",
-		"ci:test": "npm run test:report ; t1=$? ; npm run test:copyresults ; exit $t1",
+		"ci:build": "npm run build",
+		"ci:eslint": "pnpm run -r --no-sort --stream --no-bail eslint",
+		"ci:test": "npm run test:mocha:report",
 		"ci:test:coverage": "npm run test:coverage ; t1=$? ; npm run test:copyresults ; exit $t1",
-		"clean": "lerna run clean --stream --parallel && npm run clean:docs && npm run clean:nyc",
+		"clean": "pnpm run -r --no-sort --stream clean && npm run clean:docs && npm run clean:nyc",
 		"clean:docs": "rimraf --glob \"**/_api-extractor-temp\" \"docs/api/*/**\"",
 		"clean:nyc": "rimraf --glob \"nyc/**\"",
 		"commit": "git-cz",
 		"format": "npm run prettier:fix",
 		"preinstall": "node ../scripts/only-pnpm.cjs",
 		"install:commitlint": "npm install --global @commitlint/config-conventional",
-		"lint": "npm run prettier && npm run ci:eslint",
-		"lint:fix": "lerna run lint:fix --no-sort --stream",
+		"lint": "npm run syncpack:deps && npm run syncpack:versions && npm run prettier && npm run ci:eslint",
+		"lint:fix": "npm run syncpack:deps:fix && npm run syncpack:versions:fix && npm run prettier:fix && pnpm run -r --no-sort --stream lint:fix",
 		"policy-check": "node packages/build-cli/bin/dev check policy",
 		"policy-check:asserts": "node packages/build-cli/bin/dev check policy --handler assert-short-codes --fix",
 		"policy-check:fix": "node packages/build-cli/bin/dev check policy --excludeHandler assert-short-codes --fix",
@@ -43,14 +41,13 @@
 		"test": "npm run test:mocha",
 		"test:bail": "npm run test:mocha:bail",
 		"test:copyresults": "copyfiles --exclude \"**/node_modules/**\" \"**/nyc/**\" nyc",
-		"test:coverage": "c8 npm run test:report",
-		"test:mocha": "lerna run test:mocha --stream --no-bail --no-sort",
-		"test:mocha:bail": "lerna run test:mocha --stream",
-		"test:mocha:report": "lerna run test:mocha --stream --no-bail --no-sort -- -- --reporter xunit --reporter-option output=nyc/mocha-junit-report.xml",
-		"test:report": "npm run test:mocha:report",
-		"tsc": "lerna run tsc --stream",
-		"tsc:fast": "fluid-build --root .. -t tsc --buildTools",
-		"watch": "lerna run --parallel tsc -- -- --watch"
+		"test:coverage": "c8 npm run test:mocha:report",
+		"test:mocha": "pnpm run -r --no-sort --stream --no-bail test:mocha --color",
+		"test:mocha:bail": "pnpm run -r --no-sort --stream test:mocha",
+		"test:mocha:report": "pnpm run -r --no-sort --stream --no-bail test:mocha:multireport -- --timeout 4s",
+		"tsc": "fluid-build --task tsc",
+		"tsc:fast": "fluid-build --root . --task tsc --worker",
+		"watch": "pnpm run -r --parallel tsc --watch"
 	},
 	"nyc": {
 		"all": true,
@@ -69,8 +66,9 @@
 		"@commitlint/cli": "^17.6.6",
 		"@commitlint/config-conventional": "^17.6.6",
 		"@commitlint/cz-commitlint": "^17.5.0",
+		"@fluid-tools/build-cli": "^0.22.0",
 		"@fluidframework/build-common": "^2.0.0",
-		"@fluidframework/build-tools": "^0.21.0",
+		"@fluidframework/build-tools": "^0.22.0",
 		"@microsoft/api-documenter": "^7.22.24",
 		"@microsoft/api-extractor": "^7.36.1",
 		"c8": "^7.14.0",
@@ -82,7 +80,6 @@
 		"cz-conventional-changelog": "^3.3.0",
 		"cz-customizable": "^7.0.0",
 		"inquirer": "^8.2.5",
-		"lerna": "^5.6.2",
 		"prettier": "~2.6.2",
 		"rimraf": "^4.4.1",
 		"run-script-os": "^1.1.6",

common/lib/common-definitions/package.json

@@ -20,7 +20,6 @@
 		"build:compile": "concurrently npm:typetests:gen npm:tsc npm:build:esnext",
 		"build:docs": "api-extractor run --local --typescript-compiler-folder ./node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
 		"build:esnext": "tsc --project ./tsconfig.esnext.json",
-		"bump-version": "lerna version minor --no-push --no-git-tag-version && npm run build:gen:bump",
 		"ci:build": "npm run build:compile",
 		"ci:build:docs": "api-extractor run --typescript-compiler-folder ./node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
 		"ci:test": "echo No test for this package",

package.json

@@ -14,11 +14,11 @@
 		"build": "npm run policy-check && npm run layer-check && fluid-build --task build",
 		"build:compile": "fluid-build --task compile",
 		"build:docs": "fluid-build --task build:docs",
+		"build:eslint": "pnpm run -r --no-sort --stream --no-bail eslint",
 		"build:fast": "fluid-build --task build --worker",
 		"build:full": "fluid-build --task full",
 		"build:full:compile": "fluid-build --task compile --task webpack",
 		"build:gendocs": "copyfiles server/routerlicious/_api-extractor-temp/** ./_api-extractor-temp/doc-models/ -f -V && cd docs && npm run build",
-		"bump-version": "fluid-bump-version --root .",
 		"bundle-analysis:collect": "npm run webpack:profile && flub generate bundleStats",
 		"bundle-analysis:run": "flub run bundleStats --dangerfile build-tools/packages/build-cli/lib/lib/dangerfile.js",
 		"changeset": "flub changeset add",
@@ -49,8 +49,8 @@
 		"format:repo": "npm run prettier:repo:fix",
 		"preinstall": "node scripts/only-pnpm.cjs",
 		"layer-check": "flub check layers --info layerInfo.json",
-		"lint": "npm run syncpack:deps && npm run syncpack:versions && npm run prettier:root && npm run ci:eslint",
-		"lint:fix": "npm run prettier:root:fix && pnpm run -r --no-sort --stream lint:fix",
+		"lint": "npm run syncpack:deps && npm run syncpack:versions && npm run prettier:root && npm run build:eslint",
+		"lint:fix": "npm run syncpack:deps:fix && npm run syncpack:versions:fix && npm run prettier:root:fix && pnpm run -r --no-sort --stream lint:fix",
 		"policy-check": "flub check policy",
 		"policy-check-help": "echo Detect (and error on) policy-check violations, like package.json sorting, copyright headers etc. Excludes assert-short-code. Run the check or \"pnpm flub check policy --listHandlers\" for a full list.",
 		"policy-check:asserts": "flub check policy --handler assert-short-codes --fix && pretty-quick",

scripts/pack-packages.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# This script is used in the "npm pack" step in our CI build pipelines.
+# It runs (p)npm pack for all packages and sorts them into scoped/unscoped folders.
+# It also outputs a packagePublishOrder.txt file that contains the order that the packages should be published in.
+
+echo PACKAGE_MANAGER=$PACKAGE_MANAGER
+echo PUBLISH_NON_SCOPED=$PUBLISH_NON_SCOPED
+echo RELEASE_GROUP=$RELEASE_GROUP
+echo STAGING_PATH=$STAGING_PATH
+
+mkdir $STAGING_PATH/pack/
+mkdir $STAGING_PATH/pack/scoped/
+mkdir $STAGING_PATH/test-files/
+
+if [[ "$PUBLISH_NON_SCOPED" == "True" ]]; then
+  mkdir $STAGING_PATH/pack/non-scoped/
+fi
+
+if [ -f ".releaseGroup" ]; then
+  flub exec --no-private --concurrency=1 --releaseGroup $RELEASE_GROUP -- "$PACKAGE_MANAGER pack" && \
+  flub exec --no-private --concurrency=1 --releaseGroup $RELEASE_GROUP -- "mv -t $STAGING_PATH/pack/scoped/ ./*.tgz" && \
+  flub exec --no-private --releaseGroup $RELEASE_GROUP -- "[ ! -f ./*test-files.tar ] || (echo 'test files found' && mv -t $STAGING_PATH/test-files/ ./*test-files.tar)"
+
+  # This saves a list of the packages in the working directory in topological order to a temporary file.
+  # Each package name is modified to match the packed tar files.
+  flub list --no-private --releaseGroup $RELEASE_GROUP --tarball > $STAGING_PATH/pack/packagePublishOrder.txt
+else
+  $PACKAGE_MANAGER pack && mv -t $STAGING_PATH/pack/scoped/ ./*.tgz
+fi

server/gitrest/.releaseGroup

@@ -0,0 +1 @@
+The presence of this file in a directory indicates it is the root of a release group.

server/historian/.releaseGroup

@@ -0,0 +1 @@
+The presence of this file in a directory indicates it is the root of a release group.

server/routerlicious/.releaseGroup

@@ -0,0 +1 @@
+The presence of this file in a directory indicates it is the root of a release group.

tools/pipelines/templates/build-npm-package.yml

@@ -385,28 +385,15 @@ stages:
         - ${{ if ne(parameters.taskPack, false) }}:
           - task: Bash@3
             displayName: npm pack
+            env:
+              PACKAGE_MANAGER: ${{ parameters.packageManager }}
+              PUBLISH_NON_SCOPED: ${{ variables.publishNonScopedPackages }}
+              RELEASE_GROUP: ${{ parameters.tagName }}
+              STAGING_PATH: $(Build.ArtifactStagingDirectory)
             inputs:
-              targetType: 'inline'
+              targetType: 'filePath'
               workingDirectory: ${{ parameters.buildDirectory }}
-              script: |
-                mkdir $(Build.ArtifactStagingDirectory)/pack/
-                mkdir $(Build.ArtifactStagingDirectory)/pack/scoped/
-                mkdir $(Build.ArtifactStagingDirectory)/test-files/
-                if [[ "${{ variables.publishNonScopedPackages }}" == "True" ]]; then
-                  mkdir $(Build.ArtifactStagingDirectory)/pack/non-scoped/
-                fi
-                if [ -f "lerna.json" ]; then
-                  npx lerna@5.6.2 exec --no-private --no-sort -- ${{ parameters.packageManager }} pack && \
-                  npx lerna@5.6.2 exec --no-private --no-sort --parallel -- mv -t $(Build.ArtifactStagingDirectory)/pack/scoped/ ./*.tgz && \
-                  npx lerna@5.6.2 exec --no-private --no-sort --parallel -- "[ ! -f ./*test-files.tar ] || (echo 'test files found' && mv -t $(Build.ArtifactStagingDirectory)/test-files/ ./*test-files.tar)"
-
-                  # This saves a list of the packages in the working directory in topological order to a temporary file.
-                  # Each package name is modified to match the packed tar files.
-                  npx lerna@5.6.2 ls --toposort | sed 's/@//' | sed 's/\//-/' | head -c -1 > $(Build.ArtifactStagingDirectory)/pack/packagePublishOrder.txt
-                else
-                  ${{ parameters.packageManager }} pack && \
-                  mv -t $(Build.ArtifactStagingDirectory)/pack/scoped/ ./*.tgz
-                fi
+              filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh
 
           - ${{ if eq(variables.publishNonScopedPackages, true) }}:
             - ${{ each parameter in parameters.nonScopedPackages }}: