Make shared opaque directories in selfhost amenable to lazy scrubbing

Related work items: #1608015
Merged PR 524521

Public/Sdk/Experimental/NodeJs/npm.dsc

@@ -41,13 +41,8 @@ namespace Npm {
         };
     }
 
-    export interface NpmInstallResult {
-        installDir: OpaqueDirectory;
-        newPackageLock: DerivedFile;
-    }
-
     @@public
-    export function npmInstall(rootDir: StaticDirectory, packageLock: DerivedFile): NpmInstallResult {
+    export function npmInstall(rootDir: StaticDirectory, dependencies: Transformer.InputArtifact[]): OpaqueDirectory {
         const wd = rootDir.root;
         const nodeModulesPath = d`${wd}/node_modules`;
         const npmCachePath = Context.getNewOutputDirectory('npm-install-cache');
@@ -62,9 +57,8 @@ namespace Npm {
         const result = Node.run({
             arguments: arguments,
             workingDirectory: wd,
-            dependencies: [ rootDir ],
+            dependencies: [ rootDir, ...dependencies ],
             outputs: [
-                { artifact: packageLock, existence: "required" }, // rewritten file in place
                 { directory: wd, kind: "shared" },
                 npmCachePath, // Place the cache path as an output directory so it is cleaned each time.
             ],
@@ -75,10 +69,7 @@ namespace Npm {
             ],
         });
 
-        return {
-            installDir: result.getOutputDirectory(wd),
-            newPackageLock: result.getOutputFile(packageLock.path)
-        };
+        return result.getOutputDirectory(wd);
     }
 
     @@public

Public/Sdk/Public/Deployment/filesystem.dsc

@@ -99,11 +99,11 @@ export function createDeployableOpaqueSubDirectory(opaque: OpaqueDirectory, sub?
 }
 
 /**
- * Schedules a platform-specific process to copy file from 'source' to 'target'.  This process takes a dependency 
- * on 'sourceOpaqueDir`, which should be the parent opaque directory containing the file 'source'.
+ * Schedules a platform-specific process to copy file from 'source' to 'target'.
  */
 @@public
-export function copyFileFromOpaqueDirectory(source: Path, target: Path, sourceOpaqueDir: OpaqueDirectory): DerivedFile {
+export function copyFileProcess(source: Artifact, target: Artifact, dependencies: Transformer.InputArtifact[], outputs: Transformer.Output[]): Transformer.ExecuteResult {
+    const wd = Context.getNewOutputDirectory("cp");
     const args: Transformer.ExecuteArguments = Context.getCurrentHost().os === "win"
         ? <Transformer.ExecuteArguments>{
             tool: {
@@ -111,19 +111,18 @@ export function copyFileFromOpaqueDirectory(source: Path, target: Path, sourceOp
                 dependsOnWindowsDirectories: true,
                 description: "Copy File",
             },
-            workingDirectory: d`${source.parent}`,
+            workingDirectory: d`${wd}`,
             arguments: [
                 Cmd.argument("/D"),
                 Cmd.argument("/C"),
                 Cmd.argument("copy"),
                 Cmd.argument("/Y"),
                 Cmd.argument("/V"),
-                Cmd.argument(Artifact.none(source)),
-                Cmd.argument(Artifact.output(target))
+                Cmd.argument(source),
+                Cmd.argument(target)
             ],
-            dependencies: [
-                sourceOpaqueDir
-            ]
+            dependencies: dependencies,
+            outputs: outputs
         }
         : <Transformer.ExecuteArguments>{
             tool: {
@@ -132,19 +131,47 @@ export function copyFileFromOpaqueDirectory(source: Path, target: Path, sourceOp
                 dependsOnCurrentHostOSDirectories: true,
                 prepareTempDirectory: true
             },
-            workingDirectory: d`${source.parent}`,
+            workingDirectory: d`${wd}`,
             arguments: [
                 Cmd.argument("-f"),
-                Cmd.argument(Artifact.none(source)),
-                Cmd.argument(Artifact.output(target))
+                Cmd.argument(source),
+                Cmd.argument(target)
             ],
-            dependencies: [
-                sourceOpaqueDir
-            ]
+            dependencies: dependencies,
+            outputs: outputs
         };
 
-    const result = Transformer.execute(args);
-    return result.getOutputFile(target);
+    return Transformer.execute(args);
+}
+
+/**
+ * Schedules a platform-specific process to copy file from 'source' to 'target'.  This process takes a dependency 
+ * on 'sourceOpaqueDir`, which should be the parent opaque directory containing the file 'source'.
+ */
+@@public
+export function copyFileFromOpaqueDirectory(source: Path, target: Path, sourceOpaqueDir: OpaqueDirectory): DerivedFile {
+    Contract.requires(source.isWithin(sourceOpaqueDir), "Source path must be within the source opaque directory");
+    return copyFileProcess(
+        /*source*/ Artifact.none(source),
+        /*target*/ Artifact.output(target),
+        /*dependencies*/ [ sourceOpaqueDir ],
+        /*outputs*/ []
+    ).getOutputFile(target);
+}
+
+/**
+ * Schedules a platform-specific process to copy file from 'source' to 'target'.  This process takes a dependency 
+ * on 'source' and produces a shared opaque directory at the same path as 'targetOpaqueDir'.
+ */
+@@public
+export function copyFileIntoSharedOpaqueDirectory(source: File, target: Path, targetOpaqueDir: OpaqueDirectory): OpaqueDirectory {
+    Contract.requires(target.isWithin(targetOpaqueDir), "Target path must be within the target opaque directory");
+    return copyFileProcess(
+        /*source*/ Artifact.input(source),
+        /*target*/ Artifact.none(target),
+        /*dependencies*/ [ ],
+        /*outputs*/ [{ directory: targetOpaqueDir.root, kind: "shared" }]
+    ).getOutputDirectory(targetOpaqueDir.root);
 }
 
 /**
@@ -170,7 +197,7 @@ export function copyDirectory(sourceDir: Directory, targetDir: Directory, source
                 Cmd.argument(Artifact.none(sourceDir)),
                 Cmd.argument(Artifact.none(targetDir)),
                 Cmd.argument("*.*"),
-                Cmd.argument("/MIR"), // Mirror the directory
+                Cmd.argument("/E"),   // Copy subdirectories including empty ones (but no /PURGE, i.e., don't delete dest files that no longer exist)
                 Cmd.argument("/NJH"), // No Job Header
                 Cmd.argument("/NFL"), // No File list reducing stdout processing
                 Cmd.argument("/NP"),  // Don't show per-file progress counter
@@ -195,7 +222,6 @@ export function copyDirectory(sourceDir: Directory, targetDir: Directory, source
                 Cmd.argument("-arvh"),
                 Cmd.argument(Cmd.join("", [ Artifact.none(sourceDir), '/' ])),
                 Cmd.argument(Artifact.none(targetDir)),
-                Cmd.argument("--delete"),
             ],
             dependencies: [
                 sourceDirDep

Public/Src/IDE/VsCode/BuildXL.IDE.VsCode.dsc

@@ -18,17 +18,18 @@ namespace VsCode.Client {
 
     const clientCopy: OpaqueDirectory = Deployment.copyDirectory(clientSealDir.root, Context.getNewOutputDirectory("client-copy"), clientSealDir);
 
-    const copiedPackageLock = Transformer.copyFile(
+    const clientCopy2 = Deployment.copyFileIntoSharedOpaqueDirectory(
         f`${Context.getMount("CgNpmRoot").path}/package-lock.json`,
-        p`${clientCopy}/package-lock.json`);
+        p`${clientCopy}/package-lock.json`,
+        clientCopy);
 
     @public
-    export const npmInstallResult = Npm.npmInstall(clientCopy, copiedPackageLock);
+    export const npmInstall = Npm.npmInstall(clientCopy, [ clientCopy2 ]);
 
     @@public
     export const compileOutDir: OpaqueDirectory = Node.tscCompile(
         clientCopy.root, 
-        [ clientCopy, npmInstallResult.installDir, npmInstallResult.newPackageLock ]);
+        [ clientCopy, clientCopy2, npmInstall ]);
 }
 
 namespace LanguageService.Server {
@@ -111,7 +112,7 @@ namespace LanguageService.Server {
                         },
                         {
                             subfolder: a`node_modules`,
-                            contents: [ Deployment.createDeployableOpaqueSubDirectory(VsCode.Client.npmInstallResult.installDir, r`node_modules`) ]
+                            contents: [ Deployment.createDeployableOpaqueSubDirectory(VsCode.Client.npmInstall, r`node_modules`) ]
                         },
                         {
                             subfolder: a`out`,

Public/Src/IDE/VsCode/client/src/xlg-status.ts

@@ -6,10 +6,10 @@
 import * as vscode from "vscode";
 import { DocumentColorParams } from "vscode-languageserver-protocol/lib/protocol.colorProvider.proposed";
 
-export const BxlStatusCsvFileNameSuffix = ".status.csv";
+export const BxlStatusCsvFileNameSuffix = ".csv";
 export const CmdRenderStatus = "XLG.Render.Status";
 export const DefaultColumnsOfInterest = [ "*" ];
-export const DefaultColumnsToRender = [ "Cpu Percent", "Mem Percent", "Process Running" ];
+export const DefaultColumnsToRender = [ "Cpu Percent", "Ram Percent", "Process Running" ];
 
 export function activate(context: vscode.ExtensionContext) {
     context.subscriptions.push(vscode.commands.registerCommand(CmdRenderStatus, () => {

Shared/Scripts/bxl.ps1

@@ -286,7 +286,7 @@ if ($VsNew -or $VsNewNetCore -or $VsNewNet472 -or $VsNewAll) {
 }
 
 # WARNING: CloudBuild selfhost builds do NOT use this script file. When adding a new argument below, we should add the argument to selfhost queues in CloudBuild. Please contact bxl team. 
-$AdditionalBuildXLArguments += @("/remotetelemetry", "/reuseOutputsOnDisk+", "/scrubDirectory:${enlistmentRoot}\out\objects", "/storeFingerprints", "/cacheMiss", "/enableEvaluationThrottling");
+$AdditionalBuildXLArguments += @("/remotetelemetry", "/reuseOutputsOnDisk+", "/exp:LazySODeletion", "/storeFingerprints", "/cacheMiss", "/enableEvaluationThrottling");
 $AdditionalBuildXLArguments += @("/p:[Sdk.BuildXL]useQTest=true");
 
 if (($DominoArguments -match "logsDirectory:.*").Length -eq 0 -and ($DominoArguments -match "logPrefix:.*").Length -eq 0) {

bxl.sh

@@ -68,6 +68,7 @@ function compileWithBxl() {
         --config "$MY_DIR/config.dsc" 
         /generateCgManifestForNugets:"${MY_DIR}/cg/nuget/cgmanifest.json"
         /fancyConsoleMaxStatusPips:10
+        /exp:LazySODeletion
         /nowarn:11319 # DX11319: nuget version mismatch
         "$@"
     )