[rush] Look for `:incremental` suffixed scripts in watch mode (#4960)

* [rush] Support `:incremental` scripts

* PR feedback

* Use explicit typeof check

---------

Co-authored-by: David Michon <dmichon-msft@users.noreply.github.com>

common/changes/@microsoft/rush/script-priority_2024-10-03-01-47.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Changes the behavior of phased commands in watch mode to, when running a phase `_phase:<name>` in all iterations after the first, prefer a script entry named `_phase:<name>:incremental` if such a script exists. The build cache will expect the outputs from the corresponding `_phase:<name>` script (with otherwise the same inputs) to be equivalent when looking for a cache hit.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

libraries/rush-lib/src/logic/operations/IPCOperationRunnerPlugin.ts

@@ -38,7 +38,11 @@ export class IPCOperationRunnerPlugin implements IPhasedCommandPlugin {
         before: ShellOperationPluginName
       },
       async (operations: Set<Operation>, context: ICreateOperationsContext) => {
-        const { isWatch } = context;
+        const { isWatch, isInitial } = context;
+        if (!isWatch) {
+          return operations;
+        }
+
         currentContext = context;
 
         const getCustomParameterValuesForPhase: (phase: IPhase) => ReadonlyArray<string> =
@@ -51,12 +55,22 @@ export class IPCOperationRunnerPlugin implements IPhasedCommandPlugin {
             continue;
           }
 
-          const rawScript: string | undefined = project.packageJson.scripts?.[`${phase.name}:ipc`];
+          const { scripts } = project.packageJson;
+          if (!scripts) {
+            continue;
+          }
+
+          const { name: phaseName } = phase;
+
+          const rawScript: string | undefined =
+            (!isInitial ? scripts[`${phaseName}:incremental:ipc`] : undefined) ?? scripts[`${phaseName}:ipc`];
+
           if (!rawScript) {
             continue;
           }
 
-          const commandToRun: string = formatCommand(rawScript, getCustomParameterValuesForPhase(phase));
+          const customParameterValues: ReadonlyArray<string> = getCustomParameterValuesForPhase(phase);
+          const commandToRun: string = formatCommand(rawScript, customParameterValues);
 
           const operationName: string = getDisplayName(phase, project);
           let maybeIpcOperationRunner: IPCOperationRunner | undefined = runnerCache.get(operationName);
@@ -66,7 +80,7 @@ export class IPCOperationRunnerPlugin implements IPhasedCommandPlugin {
               project,
               name: operationName,
               shellCommand: commandToRun,
-              persist: isWatch,
+              persist: true,
               requestRun: (requestor?: string) => {
                 const operationState: IOperationExecutionResult | undefined =
                   operationStatesByRunner.get(ipcOperationRunner);

libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts

@@ -13,10 +13,8 @@ import { NullOperationRunner } from './NullOperationRunner';
 import { Operation } from './Operation';
 import { OperationStatus } from './OperationStatus';
 import {
-  formatCommand,
   getCustomParameterValuesByPhase,
   getDisplayName,
-  getScriptToRun,
   initializeShellOperationRunner
 } from './ShellOperationRunnerPlugin';
 
@@ -129,11 +127,8 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
         `--shard-count="${shards}"`
       ];
 
-      const rawCommandToRun: string | undefined = getScriptToRun(project, phase.name, phase.shellCommand);
-
-      const commandToRun: string | undefined = rawCommandToRun
-        ? formatCommand(rawCommandToRun, collatorParameters)
-        : undefined;
+      const { scripts } = project.packageJson;
+      const commandToRun: string | undefined = phase.shellCommand ?? scripts?.[phase.name];
 
       operation.logFilenameIdentifier = `${baseLogFilenameIdentifier}_collate`;
       operation.runner = initializeShellOperationRunner({
@@ -141,11 +136,12 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
         project,
         displayName: collatorDisplayName,
         rushConfiguration,
-        commandToRun: commandToRun
+        commandToRun,
+        customParameterValues: collatorParameters
       });
 
       const shardOperationName: string = `${phase.name}:shard`;
-      const baseCommand: string | undefined = getScriptToRun(project, shardOperationName, undefined);
+      const baseCommand: string | undefined = scripts?.[shardOperationName];
       if (baseCommand === undefined) {
         throw new Error(
           `The project '${project.packageName}' does not define a '${phase.name}:shard' command in the 'scripts' section of its package.json`
@@ -205,14 +201,11 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
 
         const shardDisplayName: string = `${getDisplayName(phase, project)} - shard ${shard}/${shards}`;
 
-        const shardedCommandToRun: string | undefined = baseCommand
-          ? formatCommand(baseCommand, shardedParameters)
-          : undefined;
-
         shardOperation.runner = initializeShellOperationRunner({
           phase,
           project,
-          commandToRun: shardedCommandToRun,
+          commandToRun: baseCommand,
+          customParameterValues: shardedParameters,
           displayName: shardDisplayName,
           rushConfiguration
         });

libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts

@@ -19,6 +19,7 @@ export interface IOperationRunnerOptions {
   rushProject: RushConfigurationProject;
   rushConfiguration: RushConfiguration;
   commandToRun: string;
+  commandForHash: string;
   displayName: string;
   phase: IPhase;
   environment?: IEnvironment;
@@ -38,6 +39,7 @@ export class ShellOperationRunner implements IOperationRunner {
   public readonly warningsAreAllowed: boolean;
 
   private readonly _commandToRun: string;
+  private readonly _commandForHash: string;
 
   private readonly _rushProject: RushConfigurationProject;
   private readonly _rushConfiguration: RushConfiguration;
@@ -53,6 +55,7 @@ export class ShellOperationRunner implements IOperationRunner {
     this._rushProject = options.rushProject;
     this._rushConfiguration = options.rushConfiguration;
     this._commandToRun = options.commandToRun;
+    this._commandForHash = options.commandForHash;
     this._environment = options.environment;
   }
 
@@ -65,7 +68,7 @@ export class ShellOperationRunner implements IOperationRunner {
   }
 
   public getConfigHash(): string {
-    return this._commandToRun;
+    return this._commandForHash;
   }
 
   private async _executeAsync(context: IOperationRunnerContext): Promise<OperationStatus> {

libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts

@@ -31,7 +31,7 @@ function createShellOperations(
   operations: Set<Operation>,
   context: ICreateOperationsContext
 ): Set<Operation> {
-  const { rushConfiguration } = context;
+  const { rushConfiguration, isInitial } = context;
 
   const getCustomParameterValuesForPhase: (phase: IPhase) => ReadonlyArray<string> =
     getCustomParameterValuesByPhase();
@@ -44,17 +44,27 @@ function createShellOperations(
       const customParameterValues: ReadonlyArray<string> = getCustomParameterValuesForPhase(phase);
 
       const displayName: string = getDisplayName(phase, project);
+      const { name: phaseName, shellCommand } = phase;
 
-      const rawCommandToRun: string | undefined = getScriptToRun(project, phase.name, phase.shellCommand);
+      const { scripts } = project.packageJson;
 
+      // This is the command that will be used to identify the cache entry for this operation
+      const commandForHash: string | undefined = shellCommand ?? scripts?.[phaseName];
+
+      // For execution of non-initial runs, prefer the `:incremental` script if it exists.
+      // However, the `shellCommand` value still takes precedence per the spec for that feature.
       const commandToRun: string | undefined =
-        rawCommandToRun !== undefined ? formatCommand(rawCommandToRun, customParameterValues) : undefined;
+        shellCommand ??
+        (!isInitial ? scripts?.[`${phaseName}:incremental`] : undefined) ??
+        scripts?.[phaseName];
 
       operation.runner = initializeShellOperationRunner({
         phase,
         project,
         displayName,
+        commandForHash,
         commandToRun,
+        customParameterValues,
         rushConfiguration
       });
     }
@@ -69,18 +79,28 @@ export function initializeShellOperationRunner(options: {
   displayName: string;
   rushConfiguration: RushConfiguration;
   commandToRun: string | undefined;
+  commandForHash?: string;
+  customParameterValues: ReadonlyArray<string>;
 }): IOperationRunner {
-  const { phase, project, rushConfiguration, commandToRun, displayName } = options;
+  const { phase, project, commandToRun: rawCommandToRun, displayName } = options;
 
-  if (commandToRun === undefined && phase.missingScriptBehavior === 'error') {
+  if (typeof rawCommandToRun !== 'string' && phase.missingScriptBehavior === 'error') {
     throw new Error(
       `The project '${project.packageName}' does not define a '${phase.name}' command in the 'scripts' section of its package.json`
     );
   }
 
-  if (commandToRun) {
+  if (rawCommandToRun) {
+    const { rushConfiguration, commandForHash: rawCommandForHash } = options;
+
+    const commandToRun: string = formatCommand(rawCommandToRun, options.customParameterValues);
+    const commandForHash: string = rawCommandForHash
+      ? formatCommand(rawCommandForHash, options.customParameterValues)
+      : commandToRun;
+
     return new ShellOperationRunner({
-      commandToRun: commandToRun || '',
+      commandToRun,
+      commandForHash,
       displayName,
       phase,
       rushConfiguration,
@@ -96,22 +116,6 @@ export function initializeShellOperationRunner(options: {
   }
 }
 
-export function getScriptToRun(
-  rushProject: RushConfigurationProject,
-  commandToRun: string,
-  shellCommand: string | undefined
-): string | undefined {
-  const { scripts } = rushProject.packageJson;
-
-  const rawCommand: string | undefined | null = shellCommand ?? scripts?.[commandToRun];
-
-  if (rawCommand === undefined || rawCommand === null) {
-    return undefined;
-  }
-
-  return rawCommand;
-}
-
 /**
  * Memoizer for custom parameter values by phase
  * @returns A function that returns the custom parameter values for a given phase

libraries/rush-lib/src/schemas/command-line.schema.json

@@ -218,7 +218,7 @@
                 },
                 "watchPhases": {
                   "title": "Watch Phases",
-                  "description": "List *exactly* the phases that should be run in watch mode for this command. If this property is specified and non-empty, after the phases defined in the \"phases\" property run, a file watcher will be started to watch projects for changes, and will run the phases listed in this property on changed projects.",
+                  "description": "List *exactly* the phases that should be run in watch mode for this command. If this property is specified and non-empty, after the phases defined in the \"phases\" property run, a file watcher will be started to watch projects for changes, and will run the phases listed in this property on changed projects. Rush will prefer scripts named \"${phaseName}:incremental\" over \"${phaseName}\" for every iteration after the first, so you can reuse the same phase name but define different scripts, e.g. to not clean on incremental runs.",
                   "type": "array",
                   "items": {
                     "type": "string"