Merged PR 782760: Update sdk.symbol to allow users to pass generated symbolMetadata file for sending symbol request

office decide to put source file into symbol drop so that source file can be retrieved from symbol store by a custom key. This will simply the implementation of the source link for office.

currently, symbolmetadata file which contains the debugentryData (including the clientKey to label the file and used for retrieving from sym store) is generated by buildxl before sending the symbol request. The key is generated based on the metadata of file by `SymstoreUtil`.

When office wants to index source files to symbol drop, source file extension may not be compatible with symstoreUtil nor do it meet requirements that key for downloading from symbolStore should be irrirelevant to content.

Office wants that `clientKey` is only relevant to path, commit or version that build can easily to compose download url for each indexed source file in symbol drop.

To do, office needs to control the generation of symbolMetaDataFile instead of letting bxl to do it.

Public/Src/Tools/SymbolDaemon/SymbolDaemon.cs

@@ -116,6 +116,14 @@ namespace Tool.SymbolDaemon
             IsMultiValue = false,
         };
 
+        internal static readonly StrOption ClientKeyForSymbol = new StrOption("customClientKey")
+        {
+            ShortName = "ck",
+            HelpText = "Custom client key for indexFiles. When this is provided, the logic for generate clientkey to files should be bypassed. This option only works for IndexFilesCmd",
+            IsRequired = false,
+            IsMultiValue = true,
+        };
+
         internal static readonly StrOption InputDirectoriesContent = new StrOption("inputDirectoriesContent")
         {
             ShortName = "idc",
@@ -330,7 +338,7 @@ namespace Tool.SymbolDaemon
         internal static readonly Command IndexFilesCmd = RegisterCommand(
             name: "indexFiles",
             description: "Indexes the specified files and saves SymbolData into a file.",
-            options: new Option[] { File, HashOptional, SymbolMetadataFile },
+            options: new Option[] { File, HashOptional, SymbolMetadataFile, ClientKeyForSymbol },
             needsIpcClient: false,
             clientAction: (ConfiguredCommand conf, IClient rpc) =>
             {
@@ -339,10 +347,11 @@ namespace Tool.SymbolDaemon
                 // hashes are sent from BXL by serializing FileContentInfo
                 var hashesWithLength = HashOptional.GetValues(conf.Config);
                 var hashesOnly = hashesWithLength.Select(h => FileContentInfo.Parse(h).Hash).ToArray();
+                var customClientKeys = ClientKeyForSymbol.GetValues(conf.Config).ToArray();
 
                 var outputFile = SymbolMetadataFile.GetValue(conf.Config);
 
-                IndexFilesAndStoreMetadataToFile(files, hashesOnly, outputFile);
+                IndexFilesAndStoreMetadataToFile(files, hashesOnly, outputFile, customClientKeys);
 
                 return 0;
             });
@@ -388,7 +397,7 @@ namespace Tool.SymbolDaemon
 
                 var outputFile = SymbolMetadataFile.GetValue(conf.Config);
 
-                IndexFilesAndStoreMetadataToFile(files.ToArray(), hashes.ToArray(), outputFile);
+                IndexFilesAndStoreMetadataToFile(files.ToArray(), hashes.ToArray(), outputFile, customClientKeys: null);
 
                 return 0;
             });
@@ -408,11 +417,16 @@ namespace Tool.SymbolDaemon
                 return SuccessOrFirstError(result);
             });
 
-        private static void IndexFilesAndStoreMetadataToFile(string[] files, ContentHash[] hashes, string outputFile)
+        private static void IndexFilesAndStoreMetadataToFile(string[] files, ContentHash[] hashes, string outputFile, string[] customClientKeys)
         {
             Contract.Assert(files.Length == hashes.Length, "Array lengths must match.");
             Contract.Assert(!string.IsNullOrEmpty(outputFile), "Output file path must be provided.");
 
+            if(customClientKeys?.Length > 0)
+            {
+                Contract.Assert(files.Length == customClientKeys.Length, "customClientKeys length must equal files length.");
+            }
+
             var indexer = new SymbolIndexer(SymbolAppTraceSource.SingleInstance);
             var symbolsMetadata = new Dictionary<ContentHash, HashSet<DebugEntryData>>(files.Length);
 
@@ -429,8 +443,26 @@ namespace Tool.SymbolDaemon
                 // of them mainly for DFA purposes). Since there is no actual file, there is nothing for us to index.
                 if (!WellKnownContentHashUtilities.IsAbsentFileHash(hashes[i]))
                 {
+                    var debugEntries = Enumerable.Empty<DebugEntryData>();
+
+                    // customClientKeys can have null/empty entries implying that BXL should generate a client key.
+                    if (customClientKeys?.Length > 0 && !string.IsNullOrEmpty(customClientKeys[i]))
+                    {
+                        debugEntries = [
+                            new DebugEntryData()
+                            {
+                                InformationLevel = (int)DebugInformationLevel.None,
+                                ClientKey = customClientKeys[i],
+                                BlobIdentifier = null
+                            }
+                        ];
+                    }
+                    else
+                    {
+                        debugEntries = indexer.GetDebugEntries(new System.IO.FileInfo(files[i]), calculateBlobId: false);
+                    }
                     // Index the file. It might not contain any symbol data. In this case, we will have an empty set.
-                    symbols.UnionWith(indexer.GetDebugEntries(new System.IO.FileInfo(files[i]), calculateBlobId: false));
+                    symbols.UnionWith(debugEntries);
                 }
             }
 

Public/Src/Tools/SymbolDaemon/Tool.SymbolDaemonInterfaces.dsc

@@ -165,7 +165,7 @@ export interface SymbolRunner {
     /** 
      * Indexes and publishes files.
      */
-    addFilesToSymbol: (createResult: SymbolCreateResult, args: OperationArguments, fileInfos: File[]) => Result;
+    addFilesToSymbol: (createResult: SymbolCreateResult, args: OperationArguments, fileInfos: File[], clientKeys?: string[]) => Result;
 
     /**
      * Indexes and publishes files withing the specified directories.

Public/Src/Tools/SymbolDaemon/Tool.SymbolDaemonRunner.dsc

@@ -156,7 +156,7 @@ function createSymbol(args: SymbolCreateArguments): SymbolCreateResult {
     return <SymbolCreateResult>{serviceStartInfo: symbolStartResult, outputs: result.outputs};
 }
 
-function addFiles(createResult: SymbolCreateResult, args: OperationArguments, files: File[]): Result {
+function addFiles(createResult: SymbolCreateResult, args: OperationArguments, files: File[], clientKeys?: string[]): Result {
     Contract.requires(
         files !== undefined,
         "files to add to a symbol request must be defined"
@@ -170,7 +170,7 @@ function addFiles(createResult: SymbolCreateResult, args: OperationArguments, fi
         return undefined;
     }
 
-    const symbolMetadataFile = indexSymbolFiles(files, args);
+    const symbolMetadataFile = indexSymbolFiles(files, args, clientKeys);
 
     const fileMessageBody = files.length !== 0
         ? [
@@ -238,7 +238,7 @@ function addDirectories(createResult: SymbolCreateResult, args: OperationArgumen
     );
 }
 
-function indexSymbolFiles(files: File[], args: OperationArguments) : DerivedFile {
+function indexSymbolFiles(files: File[], args: OperationArguments, clientKeys?: string[]) : DerivedFile {
     const symbolDataFileName = "symbol_data.txt";
 
     Contract.requires(
@@ -260,6 +260,7 @@ function indexSymbolFiles(files: File[], args: OperationArguments) : DerivedFile
             Cmd.startUsingResponseFile(false),
             Cmd.options("--file ", files.map(f => Artifact.input(f))),
             Cmd.options("--hash ", files.map(f => Artifact.vsoHash(f))),
+            Cmd.options("--customClientKey ", clientKeys),
             Cmd.option("--symbolMetadata ", Artifact.output(outputPath)),
         ],
         unsafe: {