Merged PR 545943: Dedup drop

Related work items: #1374816

Private/InternalSdk/Drop/Microsoft.Azure.Storage.Common.dsc

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import {Transformer} from "Sdk.Transformers";
+import * as Managed from "Sdk.Managed";
+
+// This is an empty facade for a Microsoft internal package.
+
+namespace Contents {
+    export declare const qualifier: {
+    };
+
+    @@public
+    export const all: StaticDirectory = Transformer.sealPartialDirectory(d`.`, []);
+}
+
+@@public
+export const pkg: Managed.ManagedNugetPackage =
+    Managed.Factory.createNugetPackage(
+        "Microsoft.Azure.Storage.Common",
+        "0.0.0",
+        Contents.all,
+        [],
+        [],
+        []
+    );

Private/InternalSdk/Drop/Microsoft.VisualStudio.Services.BlobStore.Client.Cache.dsc

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import {Transformer} from "Sdk.Transformers";
+import * as Managed from "Sdk.Managed";
+
+// This is an empty facade for a Microsoft internal package.
+
+namespace Contents {
+    export declare const qualifier: {
+    };
+
+    @@public
+    export const all: StaticDirectory = Transformer.sealPartialDirectory(d`.`, []);
+}
+
+@@public
+export const pkg: Managed.ManagedNugetPackage =
+    Managed.Factory.createNugetPackage(
+        "Microsoft.VisualStudio.Services.BlobStore.Client.Cache",
+        "0.0.0",
+        Contents.all,
+        [],
+        [],
+        []
+    );

Private/InternalSdk/Drop/module.config.dsc

@@ -48,3 +48,15 @@ module({
     nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences,
     projects: [f`Microsoft.Windows.Debuggers.SymstoreInterop.dsc`]
 });
+
+module({
+    name: "Microsoft.VisualStudio.Services.BlobStore.Client.Cache", 
+    nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences,
+    projects: [f`Microsoft.VisualStudio.Services.BlobStore.Client.Cache.dsc`]
+});
+
+module({
+    name: "Microsoft.Azure.Storage.Common", 
+    nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences,
+    projects: [f`Microsoft.Azure.Storage.Common.dsc`]
+});

Public/Sdk/SelfHost/BuildXL/BuildXLSdk.dsc

@@ -271,8 +271,8 @@ export function executable(args: Arguments): Managed.Assembly {
                 name: "Newtonsoft.Json",
                 publicKeyToken: "30ad4fe6b2a6aeed",
                 culture: "neutral",
-                oldVersion: "0.0.0.0-11.0.0.0",
-                newVersion: "11.0.0.0",
+                oldVersion: "0.0.0.0-12.0.0.0",
+                newVersion: "12.0.0.0",
             },
             {
                 name: "Microsoft.VstsContentStore",

Public/Src/App/Bxl/Program.cs

@@ -32,7 +32,6 @@ namespace BuildXL
     {
         internal const string BuildXlAppServerConfigVariable = "BUILDXL_APP_SERVER_CONFIG";
 
-        [STAThread]
         [SuppressMessage("Microsoft.Maintainability", "CA1506:AvoidExcessiveClassCoupling")]
         public static int Main(string[] rawArgs)
         {

Public/Src/Cache/ContentStore/Hashing/ComChunker.cs

@@ -38,7 +38,6 @@ namespace BuildXL.Cache.ContentStore.Hashing
         private static readonly Guid IteratorComGuid = new Guid("90B584D3-72AA-400F-9767-CAD866A5A2D8");
         private readonly IDedupIterateChunksHash32 _chunkHashIterator;
         private IDedupChunkLibrary _chunkLibrary;
-        private bool _pushBufferCalled;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="Chunker"/> class.
@@ -48,6 +47,10 @@ namespace BuildXL.Cache.ContentStore.Hashing
             _chunkLibrary = NativeMethods.CreateChunkLibrary();
             _chunkLibrary.InitializeForPushBuffers();
 
+            Contract.Assert(
+                Thread.CurrentThread.GetApartmentState() == ApartmentState.MTA,
+                "Thread must be in MTA ApartmentState");
+
             object chunksEnum;
             _chunkLibrary.StartChunking(IteratorComGuid, out chunksEnum);
             _chunkHashIterator = (IDedupIterateChunksHash32)chunksEnum;
@@ -56,19 +59,9 @@ namespace BuildXL.Cache.ContentStore.Hashing
         /// <inheritdoc/>
         public IChunkerSession BeginChunking(Action<ChunkInfo> chunkCallback)
         {
-            Reset();
             return new Session(this, chunkCallback);
         }
 
-        /// <summary>
-        /// Reinitializes this instance for reuse.
-        /// </summary>
-        private void Reset()
-        {
-            _pushBufferCalled = false;
-            _chunkHashIterator.Reset();
-        }
-
         /// <summary>
         /// Chunks the buffer, calling back when chunks complete.
         /// </summary>
@@ -94,13 +87,6 @@ namespace BuildXL.Cache.ContentStore.Hashing
                 throw new IndexOutOfRangeException();
             }
 
-            if (_pushBufferCalled)
-            {
-                throw new InvalidOperationException("PushBuffer can only be called once.");
-            }
-
-            _pushBufferCalled = true;
-
             fixed (byte* ptr = &buffer[startOffset])
             {
                 _chunkHashIterator.PushBuffer(ptr, (uint)count);
@@ -114,11 +100,9 @@ namespace BuildXL.Cache.ContentStore.Hashing
         /// </summary>
         private unsafe void DonePushing(Action<ChunkInfo> chunkCallback)
         {
-            if (_pushBufferCalled)
-            {
-                _chunkHashIterator.Drain();
-                ProcessChunks(chunkCallback);
-            }
+            _chunkHashIterator.Drain();
+            ProcessChunks(chunkCallback);
+            _chunkHashIterator.Reset();
         }
 
         /// <inheritdoc/>
@@ -133,8 +117,6 @@ namespace BuildXL.Cache.ContentStore.Hashing
 
         private void ProcessChunks(Action<ChunkInfo> chunkCallback)
         {
-            Contract.Assert(_pushBufferCalled);
-
             uint ulFetchedChunks;
             do
             {
@@ -166,7 +148,7 @@ namespace BuildXL.Cache.ContentStore.Hashing
         }
 
         /// <inheritdoc/>
-        public readonly struct Session : IChunkerSession, IDisposable
+        private sealed class Session : IChunkerSession, IDisposable
         {
             private readonly ComChunkerNonDeterministic _chunker;
             private readonly Action<ChunkInfo> _chunkCallback;
@@ -187,38 +169,7 @@ namespace BuildXL.Cache.ContentStore.Hashing
             /// <inheritdoc/>
             public void Dispose()
             {
-                try
-                {
-                    _chunker.DonePushing(_chunkCallback);
-                }
-                catch (COMException e) when ((uint)e.ErrorCode == 0x80565319)
-                {
-                    // Maybe in in an "invalid state"
-                }
-            }
-
-            /// <inheritdoc/>
-            public override bool Equals(object obj)
-            {
-                throw new InvalidOperationException();
-            }
-
-            /// <inheritdoc/>
-            public override int GetHashCode()
-            {
-                throw new InvalidOperationException();
-            }
-
-            /// <nodoc />
-            public static bool operator ==(Session left, Session right)
-            {
-                throw new InvalidOperationException();
-            }
-
-            /// <nodoc />
-            public static bool operator !=(Session left, Session right)
-            {
-                throw new InvalidOperationException();
+                _chunker.DonePushing(_chunkCallback);
             }
         }
 
@@ -377,8 +328,8 @@ namespace Microsoft.DataDeduplication.Interop
         /// <nodoc />
         [MarshalAs(UnmanagedType.ByValArray, SizeConst = 32)]
         public byte[] Hash; // 32-byte chunk hash value
-    } 
-    
+    }
+
     /// <nodoc />
     [InterfaceType(ComInterfaceType.InterfaceIsIUnknown), Guid("90B584D3-72AA-400F-9767-CAD866A5A2D8")]
     unsafe public interface IDedupIterateChunksHash32

Public/Src/Cache/ContentStore/Hashing/DedupNodeHashAlgorithm.cs

@@ -13,21 +13,33 @@ namespace BuildXL.Cache.ContentStore.Hashing
     /// </summary>
     public sealed class DedupNodeHashAlgorithm : HashAlgorithm
     {
+        /// <nodoc />
+        public static Lazy<Exception> ComChunkerLoadError = new Lazy<Exception>(() =>
+        {
+            try
+            {
+                var chunker = new ComChunker();
+                using var session = chunker.BeginChunking(chunk => { });
+                var content = new byte[1024 * 1024 + 1];
+                session.PushBuffer(content, 0, content.Length);
+                return null;
+            }
+            catch (Exception e)
+            {
+#pragma warning disable ERP022 // Unobserved exception in generic exception handler
+                return e;
+#pragma warning restore ERP022 // Unobserved exception in generic exception handler
+            }
+        });
+
         /// <summary>
         /// Creates a chunker appropriate to the runtime environment
         /// </summary>
         public static IChunker CreateChunker()
         {
-            if (IsComChunkerSupported)
+            if (IsComChunkerSupported && ComChunkerLoadError.Value == null)
             {
-                try
-                {
-                    return new ComChunker();
-                }
-                catch (System.ComponentModel.Win32Exception)
-                {
-                    // Some older versions of windows. Fall back to managed chunker.
-                }
+                return new ComChunker();
             }
 
             return new ManagedChunker();

Public/Src/Cache/ContentStore/Hashing/DedupNodeOrChunkHashAlgorithm.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System;
 using System.Collections.Generic;
 using System.Diagnostics.ContractsLight;
 using System.Linq;
@@ -51,12 +52,12 @@ namespace BuildXL.Cache.ContentStore.Hashing
         {
             _chunks.Clear();
             _session = _chunker.BeginChunking(SaveChunks);
-            _lastNode = null;
         }
 
         /// <inheritdoc />
         protected override void HashCore(byte[] array, int ibStart, int cbSize)
         {
+            _lastNode = null;
             _session.PushBuffer(array, ibStart, cbSize);
         }
 

Public/Src/Cache/ContentStore/Hashing/DeterministicChunker.cs

@@ -30,7 +30,7 @@ namespace BuildXL.Cache.ContentStore.Hashing
             _chunker.Dispose();
         }
 
-        private class Session : IChunkerSession
+        private sealed class Session : IChunkerSession
         {
             private static readonly ByteArrayPool PushBufferPool = new ByteArrayPool((int)Chunker.MinPushBufferSize);
             private static readonly Pool<List<ChunkInfo>> ChunksSeenPool = new Pool<List<ChunkInfo>>(() => new List<ChunkInfo>(), list => list.Clear());

Public/Src/Cache/ContentStore/Vsts/DedupContentSession.cs

@@ -17,6 +17,7 @@ using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Utilities.Tracing;
 using Microsoft.VisualStudio.Services.BlobStore.Common;
 using Microsoft.VisualStudio.Services.BlobStore.WebApi;
+using Microsoft.VisualStudio.Services.Content.Common;
 using VstsDedupIdentifier = Microsoft.VisualStudio.Services.BlobStore.Common.DedupIdentifier;
 using VstsFileSystem = Microsoft.VisualStudio.Services.Content.Common.FileSystem;
 
@@ -84,7 +85,7 @@ namespace BuildXL.Cache.ContentStore.Vsts
                     return new PutResult(pinResult, contentHash);
                 }
 
-                var dedupNode = await GetDedupNodeFromFileAsync(path.Path, context.Token);
+                var dedupNode = await GetDedupNodeFromFileAsync(path.Path, _artifactFileSystem, context.Token);
                 var calculatedHash = dedupNode.ToContentHash();
 
                 if (contentHash != calculatedHash)
@@ -130,7 +131,7 @@ namespace BuildXL.Cache.ContentStore.Vsts
             try
             {
                 var contentSize = GetContentSize(path);
-                var dedupNode = await GetDedupNodeFromFileAsync(path.Path, context.Token);
+                var dedupNode = await GetDedupNodeFromFileAsync(path.Path, _artifactFileSystem, context.Token);
                 var contentHash = dedupNode.ToContentHash();
 
                 if (contentHash.HashType != hashType)
@@ -279,10 +280,10 @@ namespace BuildXL.Cache.ContentStore.Vsts
                                 innerCts));
         }
 
-        private async Task<DedupNode> GetDedupNodeFromFileAsync(string path, CancellationToken token)
+        internal static async Task<DedupNode> GetDedupNodeFromFileAsync(string path, IFileSystem fileSystem, CancellationToken token)
         {
             var dedupNode = await ChunkerHelper.CreateFromFileAsync(
-                fileSystem: _artifactFileSystem,
+                fileSystem: fileSystem,
                 path: path,
                 cancellationToken: token,
                 configureAwait: false);

Public/Src/Cache/MemoizationStore/Vsts/BuildXL.Cache.MemoizationStore.Vsts.dsc

@@ -20,7 +20,7 @@ namespace Vsts {
             ContentStore.Interfaces.dll,
             ContentStore.Library.dll,
             ContentStore.Vsts.dll,
-            importFrom("Newtonsoft.Json.v10").pkg,
+            importFrom("Newtonsoft.Json").pkg,
             importFrom("System.Interactive.Async").pkg,
             importFrom("Microsoft.VisualStudio.Services.Client").pkg,
             BuildXLSdk.Factory.createBinary(importFrom("TransientFaultHandling.Core").Contents.all, r`/lib/NET4/Microsoft.Practices.TransientFaultHandling.Core.dll`),

Public/Src/Cache/MemoizationStore/VstsInterfaces/BuildXL.Cache.MemoizationStore.Vsts.Interfaces.dsc

@@ -13,7 +13,7 @@ namespace VstsInterfaces {
             ContentStore.VstsInterfaces.dll,
             Interfaces.dll,
             NetFx.System.Runtime.Serialization.dll,
-            importFrom("Newtonsoft.Json.v10").pkg,
+            importFrom("Newtonsoft.Json").pkg,
         ],
     });
 }

Public/Src/Cache/MemoizationStore/VstsTest/App.config

@@ -4,7 +4,7 @@
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
       <dependentAssembly>
         <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-11.0.0.0" newVersion="10.0.0.0" />
+        <bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
       </dependentAssembly>
     </assemblyBinding>
   </runtime>

Public/Src/Cache/MemoizationStore/VstsTest/BuildXL.Cache.MemoizationStore.Vsts.Test.dsc

@@ -29,17 +29,12 @@ namespace VstsTest {
             VstsInterfaces.dll,
             Vsts.dll,
 
-            importFrom("Newtonsoft.Json.v10").pkg,
+            importFrom("Newtonsoft.Json").pkg,
             ...importFrom("BuildXL.Cache.ContentStore").redisPackages,
             importFrom("Microsoft.VisualStudio.Services.Client").pkg,
             BuildXLSdk.visualStudioServicesArtifactServicesWorkaround,
             ...BuildXLSdk.fluentAssertionsWorkaround,
         ],
-        deploymentOptions: {
-            excludedDeployableItems: [
-            // This code uses newtonsoft v10 but depends transitively on code with the latest version. This needs to use v10, so block deployment of the latest version.
-            importFrom("Newtonsoft.Json").pkg,
-        ]},
         runtimeContent: [
             {
                 subfolder: r`redisServer`,

Public/Src/Cache/MemoizationStore/VstsTest/DedupContentSessionTests.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using BuildXL.Cache.ContentStore.Hashing;
+using BuildXL.Cache.ContentStore.Vsts;
+using FluentAssertions;
+using Microsoft.VisualStudio.Services.Content.Common;
+using Xunit;
+
+namespace BuildXL.Cache.MemoizationStore.Vsts.Test
+{
+    public class DedupContentSessionTests
+    {
+        [Theory]
+        [InlineData(0)]
+        [InlineData(1024)]
+        [InlineData(128*1024+1)] // Chunk size is 128 * 1024
+        public async Task CanGetHashFromFile(int fileLength)
+        {
+            using var stream = new MemoryStream(new byte[fileLength]);
+            var node = await DedupContentSession.GetDedupNodeFromFileAsync(string.Empty, new TestFileSystem(stream), CancellationToken.None);
+            node.HashString.Should().NotBeNullOrEmpty();
+        }
+
+        [Fact]
+        public void ComChunkerWorksOnThreading()
+        {
+            // For some reason current thread is STAThread, so we need to start a new one.
+            var thread = new Thread(() =>
+            {
+                Thread.CurrentThread.GetApartmentState().Should().Be(ApartmentState.MTA);
+                var chunker = new ComChunker();
+                Task.Run(() =>
+                {
+                    using var session = chunker.BeginChunking(chunk => { });
+                    session.PushBuffer(new ArraySegment<byte>(new byte[1]));
+                }).GetAwaiter().GetResult();
+            });
+            thread.Start();
+            thread.Join();
+        }
+
+        private class TestFileSystem : IFileSystem
+        {
+            private readonly Stream _stream;
+
+            public TestFileSystem(Stream stream)
+            {
+                _stream = stream;
+            }
+            public void CreateDirectory(string directoryPath) => throw new NotImplementedException();
+            public void DeleteFile(string filePath) => throw new NotImplementedException();
+            public bool DirectoryExists(string directoryPath) => throw new NotImplementedException();
+            public IEnumerable<string> EnumerateDirectories(string directoryFullPath, bool recursiveSearch) => throw new NotImplementedException();
+            public IEnumerable<string> EnumerateFiles(string directoryFullPath, bool recursiveSearch) => throw new NotImplementedException();
+            public bool FileExists(string filePath) => throw new NotImplementedException();
+            public long GetFileSize(string filePath) => throw new NotImplementedException();
+            public string GetRandomFileName() => throw new NotImplementedException();
+            public string GetTempFullPath() => throw new NotImplementedException();
+            public string GetWorkingDirectory() => throw new NotImplementedException();
+            public FileStream OpenFileStreamForAsync(string fileFullPath, FileMode fileMode, FileAccess fileAccess, FileShare fileShare) => throw new NotImplementedException();
+            public Stream OpenStreamForFile(string fileFullPath, FileMode fileMode, FileAccess fileAccess, FileShare fileShare) => _stream;
+            public StreamReader OpenText(string filePath) => throw new NotImplementedException();
+            public byte[] ReadAllBytes(string filePath) => throw new NotImplementedException();
+            public string ReadAllText(string filePath) => throw new NotImplementedException();
+            public void WriteAllBytes(string filePath, byte[] bytes) => throw new NotImplementedException();
+            public void WriteAllText(string filePath, string content) => throw new NotImplementedException();
+            public TempFile GetTempFileFullPath() => throw new NotImplementedException();
+        }
+    }
+}

Public/Src/Cache/Monitor/App/Scheduler.cs

@@ -380,7 +380,7 @@ namespace BuildXL.Cache.Monitor.App
                 using var stream = File.OpenText(stateFilePath);
                 var serializer = CreateSerializer();
 
-                var state = (Dictionary<string, PersistableEntry>)serializer.Deserialize(stream, typeof(Dictionary<string, PersistableEntry>));
+                var state = (Dictionary<string, PersistableEntry>?)serializer.Deserialize(stream, typeof(Dictionary<string, PersistableEntry>));
                 if (state is null)
                 {
                     // Happens when the file is empty

Public/Src/Cache/VerticalStore/BuildCacheAdapter/BuildXL.Cache.BuildCacheAdapter.dsc

@@ -35,9 +35,6 @@ namespace BuildCacheAdapter {
             BuildXLSdk.visualStudioServicesArtifactServicesWorkaround,
             ...importFrom("BuildXL.Cache.ContentStore").redisPackages,
             importFrom("WindowsAzure.Storage").pkg,
-        ],
-        runtimeContentToSkip: [
-            importFrom("Newtonsoft.Json.v10").pkg, // CloudStore has to rely on NewtonSoft.Json version 10. BuildXL and asp.net core depend on 11.
         ]
     });
 }

Public/Src/Deployment/cache.dsc

@@ -36,10 +36,5 @@ namespace Cache {
     export const deployed = BuildXLSdk.DeploymentHelpers.deploy({
         definition: deployment,
         targetLocation: r`${qualifier.configuration}/cache/${qualifier.targetFramework}/${qualifier.targetRuntime}`,
-        deploymentOptions: {
-            excludedDeployableItems: [
-                importFrom("Newtonsoft.Json.v10").pkg,
-            ]
-        }
     });
 }

Public/Src/Engine/Dll/Engine.cs

@@ -1207,6 +1207,9 @@ namespace BuildXL.Engine
                             { "VSO0:C6AB5808D30BFF857263BC467FE8D818F35486763F673F79CA5A758727CEF3A900","DEDUPNODEORCHUNK:1D9CED63701BC0F2E5D3063BA7889F8687537CB78B155858FCB7EE56A78A6C8102" },
                             { "VSO0:6BBAE77F9BA0231C90ABD9EA720FF886E8613CE8EF29D8B657AF201E2982829600","DEDUPNODEORCHUNK:1A350CECC53CAE31EE3699BDA53270E91951A81E6353EABC878BA8D8B16F8E9202" },
                             { "VSO0:EE359BDFFFED53EF3C5E76C1716AADD1567447B12A37292C075D6A26F1138C0700","DEDUPNODEORCHUNK:EEA76D4D9582AF60528262048E03F957FA7A91D166DE37D2C983A31583D8FE9F02" },
+                            { "VSO0:FCB44A9D07D3923DB197C05A710FEBBB060649555418A067E04EAE1A06CBCE4400","DEDUPNODEORCHUNK:17819CDB4B3658BC8A730DD1B1C6AFA427E3D5651789519309A7A32523F382B402" },
+                            { "VSO0:B65327703FB1775A7ABD637D44816CDE13DFE01BD98FF2B1B1DE8DAC46D1567800","DEDUPNODEORCHUNK:C4DFAA60F349EC7DADFB0BF53D0B278CFD41786FB276A25A2C94FE8724A6CF1202" },
+                            { "VSO0:B89DFFD762BEA6D94E11CEA1C430FDC620CE5407827360085B21963E5887E38300","DEDUPNODEORCHUNK:7358076CBF7C874C31B5599D0455209282539958F9F36FAF28155496442093CB02" },
                         };
 
                     foreach (var download in x.Downloads.Cast<DownloadFileSettings>())

Public/Src/Tools/ContentPlacementAnalysisTools/ContentPlacementAnalysisTools.Extraction/App.config

@@ -21,7 +21,7 @@
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
       <dependentAssembly>
         <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-11.0.0.0" newVersion="11.0.0.0" />
+        <bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
       </dependentAssembly>
     </assemblyBinding>
   </runtime>

Public/Src/Tools/DropDaemon/DropConfig.cs

@@ -67,6 +67,11 @@ namespace Tool.DropDaemon
         /// </summary>
         public string LogDir { get; }
 
+        /// <summary>
+        ///     File name for artifact-side logs
+        /// </summary>
+        public string ArtifactLogName { get; }
+
         #endregion
 
         #region Defaults
@@ -113,7 +118,8 @@ namespace Tool.DropDaemon
             bool? verbose = null,
             bool? enableTelemetry = null,
             bool? enableChunkDedup = null,
-            string logDir = null)
+            string logDir = null,
+            string artifactLogName = null)
         {
             Name = dropName;
             Service = serviceEndpoint;
@@ -124,6 +130,7 @@ namespace Tool.DropDaemon
             EnableTelemetry = enableTelemetry ?? DefaultEnableTelemetry;
             EnableChunkDedup = enableChunkDedup ?? DefaultEnableChunkDedup;
             LogDir = logDir;
+            ArtifactLogName = artifactLogName;
         }
     }
 }

Public/Src/Tools/DropDaemon/DropDaemon.cs

@@ -21,6 +21,7 @@ using BuildXL.Utilities.Instrumentation.Common;
 using BuildXL.Utilities.Tasks;
 using Microsoft.VisualStudio.Services.Common;
 using Microsoft.VisualStudio.Services.Drop.WebApi;
+using Microsoft.VisualStudio.Services.Graph;
 using Microsoft.VisualStudio.Services.WebApi;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -672,7 +673,8 @@ namespace Tool.DropDaemon
                 verbose: conf.Get(Verbose),
                 enableTelemetry: conf.Get(EnableTelemetry),
                 enableChunkDedup: conf.Get(EnableChunkDedup),
-                logDir: conf.Get(LogDir));
+                logDir: conf.Get(LogDir),
+                artifactLogName: conf.Get(ArtifactLogName));
         }
 
         private static T RegisterDropConfigOption<T>(T option) where T : Option => RegisterOption(DropConfigOptions, option);

Public/Src/Tools/DropDaemon/DropDaemon.exe.config

@@ -39,7 +39,7 @@
       </dependentAssembly>
       <dependentAssembly>
         <assemblyIdentity name="Microsoft.IdentityModel.Clients.ActiveDirectory" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-4.5.1.0" newVersion="4.5.1.0" />
+        <bindingRedirect oldVersion="0.0.0.0-5.2.6.0" newVersion="5.2.6.0" />
       </dependentAssembly>
       <dependentAssembly>
         <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />

Public/Src/Tools/DropDaemon/ReloadingDropServiceClient.cs

@@ -114,15 +114,6 @@ namespace Tool.ServicePipDaemon
                 cancellationToken);
         }
 
-        /// <inheritdoc />
-        public Task<IEnumerable<DropItem>> ListAsync(string dropNamePrefix, PathOptions pathOptions, bool includeNonFinalizedDrops, CancellationToken cancellationToken, RetrievalOptions retrievalOptions = RetrievalOptions.ExcludeSoftDeleted)
-        {
-            return RetryAsync(
-                nameof(IDropServiceClient.ListAsync),
-                (client, ct) => client.ListAsync(dropNamePrefix, pathOptions, includeNonFinalizedDrops, ct, retrievalOptions),
-                cancellationToken);
-        }
-
         /// <inheritdoc />
         public Task UploadAndAssociateAsync(string dropName, List<FileBlobDescriptor> preComputedBlobIds, bool abortIfAlreadyExists, AssociationsStatus firstAssociationStatus, CancellationToken cancellationToken)
         {
@@ -185,6 +176,15 @@ namespace Tool.ServicePipDaemon
 
         #endregion
 
+        /// <inheritdoc />
+        public Task<IEnumerable<DropItem>> ListAsync(string dropNamePrefix, PathOptions pathOptions, bool includeNonFinalizedDrops, CancellationToken cancellationToken, RetrievalOptions retrievalOptions, SizeOptions sizeOptions, ExpirationDateOptions expirationDateOptions)
+        {
+            return RetryAsync(
+                    nameof(IDropServiceClient.ListAsync),
+                    (client, ct) => client.ListAsync(dropNamePrefix, pathOptions, includeNonFinalizedDrops, ct, retrievalOptions, sizeOptions, expirationDateOptions),
+                    cancellationToken);
+        }
+
         /// <inheritdoc />
         public string GetVersionString()
         {

Public/Src/Tools/DropDaemon/Tool.DropDaemon.dsc

@@ -45,6 +45,11 @@ export namespace DropDaemon {
             importFrom("Microsoft.VisualStudio.Services.InteractiveClient").pkg,
             importFrom("Newtonsoft.Json").pkg,
             importFrom("WindowsAzure.Storage").pkg,
+            importFrom("Microsoft.Azure.Storage.Common").pkg,
+
+            // We need to reference this even though the codepath which uses the path is never activated 
+            // because of the way that runtime assemblies are loaded into memory.
+            importFrom("Microsoft.VisualStudio.Services.BlobStore.Client.Cache").pkg, 
         ],
         internalsVisibleTo: [
             "Test.Tool.DropDaemon",

Public/Src/Tools/DropDaemon/VsoClient.cs

@@ -158,6 +158,12 @@ namespace Tool.DropDaemon
             // create and set up timer for triggering the batch block
             TimeSpan timerInterval = m_config.NagleTime;
             m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval);
+
+            if (m_config.ArtifactLogName != null)
+            {
+                DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
+                Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
+            }
         }
 
         /// <summary>
@@ -257,6 +263,7 @@ namespace Tool.DropDaemon
                 new DropClientTelemetry(ServiceEndpoint, Tracer, enable: m_config.EnableTelemetry),
                 Tracer);
             Interlocked.Add(ref Stats.AuthTimeMs, ElapsedMillis(startTime));
+
             return client;
         }
 
@@ -550,7 +557,7 @@ namespace Tool.DropDaemon
                             relativePath: m_dropItem.RelativeDropPath,
                             fileSize: m_dropItem.FileLength,
                             blobId: m_dropItem.BlobIdentifier),
-                        m_dropItem.FullFilePath);
+                        absolutePath: null); // If we pass it in, the client will actually try to use the file, and we cannot be sure that it has been materialized.
                 }
                 else
                 {

Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/CacheHitPredictor/CacheHitPredictor.cs

@@ -236,7 +236,7 @@ namespace BuildXL.Execution.Analyzer
 #if NET_FRAMEWORK
             var platformParameters = new PlatformParameters(PromptBehavior.Auto);
 #else
-            var platformParameters = new PlatformParameters();
+            PlatformParameters platformParameters = null; // .NET Core does not support interactive auth.
 #endif
 
             var authenticationResult = authenticationContext.AcquireTokenAsync(Resource, Client, RedirectUri, platformParameters).GetAwaiter().GetResult();

Public/Src/Tools/Execution.Analyzer/App.config

@@ -40,7 +40,7 @@
       </dependentAssembly>
       <dependentAssembly>
         <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-11.0.0.0" newVersion="11.0.0.0" />
+        <bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
       </dependentAssembly>
       <dependentAssembly>
         <assemblyIdentity name="Microsoft.TeamFoundation.Common" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />

Public/Src/Tools/ServicePipDaemon/ServicePipDaemon.cs

@@ -168,6 +168,13 @@ namespace Tool.ServicePipDaemon
             IsRequired = false
         });
 
+        /// <nodoc />
+        public static readonly StrOption ArtifactLogName = RegisterDaemonConfigOption(new StrOption("artifactLogName")
+        {
+            HelpText = "Name for artifact-side log file",
+            IsRequired = false
+        });
+
         /// <nodoc />
         public static readonly StrOption File = new StrOption("file")
         {

Public/Src/Tools/SymbolDaemon/ReloadingSymbolClient.cs

@@ -7,6 +7,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using BuildXL.Ipc.Interfaces;
 using Microsoft.VisualStudio.Services.BlobStore.Common;
+using Microsoft.VisualStudio.Services.Content.Common;
 using Microsoft.VisualStudio.Services.Symbol.App.Core;
 using Microsoft.VisualStudio.Services.Symbol.WebApi;
 
@@ -84,11 +85,11 @@ namespace Tool.ServicePipDaemon
         }
 
         /// <inheritdoc />
-        public Task<IEnumerable<Request>> GetAllRequestsAsync(CancellationToken cancellationToken)
+        public Task<IEnumerable<Request>> GetAllRequestsAsync(CancellationToken cancellationToken, SizeOptions sizeOptions, ExpirationDateOptions expirationDateOptions)
         {
             return RetryAsync(
                 nameof(ISymbolServiceClient.GetAllRequestsAsync),
-                (client, ct) => client.GetAllRequestsAsync(ct),
+                (client, ct) => client.GetAllRequestsAsync(ct, sizeOptions, expirationDateOptions),
                 cancellationToken);
         }
 
@@ -143,11 +144,11 @@ namespace Tool.ServicePipDaemon
         }
 
         /// <inheritdoc />
-        public Task<IEnumerable<Request>> GetRequestPaginatedAsync(string continueFromRequestId, int pageSize, CancellationToken cancellationToken)
+        public Task<IEnumerable<Request>> GetRequestPaginatedAsync(string continueFromRequestId, int pageSize, CancellationToken cancellationToken, SizeOptions sizeOptions, ExpirationDateOptions expirationDateOptions)
         {
             return RetryAsync(
                  nameof(ISymbolServiceClient.GetRequestPaginatedAsync),
-                 (client, ct) => client.GetRequestPaginatedAsync(continueFromRequestId, pageSize, ct),
+                 (client, ct) => client.GetRequestPaginatedAsync(continueFromRequestId, pageSize, ct, sizeOptions, expirationDateOptions),
                  cancellationToken);
         }
 

Public/Src/Tools/UnitTests/DropDaemon/ReloadingDropServiceClientTest.cs

@@ -228,7 +228,7 @@ namespace Test.Tool.DropDaemon
             throw new NotImplementedException();
         }
 
-        Task<IEnumerable<DropItem>> IDropServiceClient.ListAsync(string dropNamePrefix, PathOptions pathOptions, bool includeNonFinalizedDrops, CancellationToken cancellationToken, RetrievalOptions retrievalOptions)
+        Task<IEnumerable<DropItem>> IDropServiceClient.ListAsync(string dropNamePrefix, PathOptions pathOptions, bool includeNonFinalizedDrops, CancellationToken cancellationToken, RetrievalOptions retrievalOptions, SizeOptions sizeOptions, ExpirationDateOptions expirationDateOptions)
         {
             throw new NotImplementedException();
         }

Public/Src/Tools/UnitTests/DropDaemon/Test.Tool.DropDaemon.dll.config

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <configSections>
+    <section name="tracerSettings" type="Microsoft.VisualStudio.Services.ArtifactServices.App.Shared.Tracing.TraceListenerConfigurationSection, Microsoft.VisualStudio.Services.ArtifactServices.App.Shared" />
+  </configSections>
+
+  <tracerSettings>
+    <!--
+      Valid TraceLevel values: info|warn|error|fatal|verbose, and other System.Diagnostics.SourceLevels values
+      Valid TraceListener values: type="console|file" file="[File path]"; 'file' only applies if Type is file, and if empty defaults to %TEMP%\[appname]_[timestamp].log
+    -->
+    <traceLevel value="Info" />
+    <listeners>
+      <add type="console" />
+    </listeners>
+  </tracerSettings>
+
+  <appSettings>
+    <add key="defaultServiceLocation" value="" />
+  </appSettings>
+
+  <startup>
+    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.1" />
+  </startup>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <probing privatePath="Microsoft.AspNet.WebApi.Core.5.2.2\lib\net45;Microsoft.AspNet.WebApi.Client.5.2.2\lib\net45" />
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.Diagnostics.Tracing.EventSource" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-1.1.16.0" newVersion="1.1.28.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.ContentStoreInterfaces" publicKeyToken="1055fbdf2d8b69e0" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-15.1250.0.0" newVersion="15.1280.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Threading.Tasks.Dataflow" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.6.3.0" newVersion="4.6.3.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.IdentityModel.Clients.ActiveDirectory" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-5.2.6.0" newVersion="5.2.6.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-9.3.2.0" newVersion="9.3.2.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+
+  <!--  turn off the default opening of a dialog for Debug.Fail.  We are sure to handle all exceptions in the
+  app and operations so it is just confusing noise to the user -->
+  <system.diagnostics>
+    <trace>
+      <listeners>
+        <remove name="Default" />
+      </listeners>
+    </trace>
+  </system.diagnostics>
+</configuration>

Public/Src/Tools/UnitTests/DropDaemon/Test.Tool.DropDaemon.dsc

@@ -7,6 +7,7 @@ namespace Test.Tool.DropDaemon {
     export const dll = !BuildXLSdk.Flags.isMicrosoftInternal ? undefined : BuildXLSdk.test({
         assemblyName: "Test.Tool.DropDaemon",
         sources: globR(d`.`, "*.cs"),
+        appConfig: f`Test.Tool.DropDaemon.dll.config`,
         references: [
             importFrom("BuildXL.Cache.ContentStore").Hashing.dll,
             importFrom("BuildXL.Cache.ContentStore").UtilitiesCore.dll,
@@ -26,9 +27,11 @@ namespace Test.Tool.DropDaemon {
             importFrom("Microsoft.AspNet.WebApi.Client").pkg,
             BuildXLSdk.visualStudioServicesArtifactServicesWorkaround,
             importFrom("Microsoft.VisualStudio.Services.BlobStore.Client").pkg,
+            importFrom("Microsoft.VisualStudio.Services.BlobStore.Client.Cache").pkg,
             importFrom("Microsoft.IdentityModel.Clients.ActiveDirectory").pkg,
             importFrom("Microsoft.VisualStudio.Services.Client").pkg,
             importFrom("Microsoft.VisualStudio.Services.InteractiveClient").pkg,
+            importFrom("Microsoft.Azure.Storage.Common").pkg,
         ]
     });
 }

Public/Src/Utilities/Storage/ContentHashingUtilities.cs

@@ -8,7 +8,9 @@ using System.Text;
 using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Native.IO;
+using BuildXL.Storage.Tracing;
 using BuildXL.Utilities;
+using BuildXL.Utilities.Tracing;
 using static BuildXL.Utilities.FormattableStringEx;
 
 namespace BuildXL.Storage
@@ -126,6 +128,25 @@ namespace BuildXL.Storage
         {
             s_isInitialized = true;
             HashInfo = HashInfoLookup.Find(hashType);
+
+            if (hashType == HashType.DedupNodeOrChunk || hashType == HashType.DedupNode)
+            {
+                if (DedupNodeHashAlgorithm.IsComChunkerSupported)
+                { 
+                    if (DedupNodeHashAlgorithm.ComChunkerLoadError.Value != null)
+                    {
+                        Logger.Log.ComChunkerFailulre(Events.StaticContext, DedupNodeHashAlgorithm.ComChunkerLoadError.Value.ToString());
+                    }
+                    else
+                    {
+                        Logger.Log.ChunkerType(Events.StaticContext, nameof(ComChunker));
+                    }
+                }
+                else
+                {
+                    Logger.Log.ChunkerType(Events.StaticContext, nameof(ManagedChunker));
+                }
+            }
         }
 
         /// <summary>

Public/Src/Utilities/Storage/Tracing/Log.cs

@@ -734,6 +734,24 @@ namespace BuildXL.Storage.Tracing
             Message = "Directory '{path}' is enumerated multiple times with different resulting directory fingerprints; this indicates that the membership of directory changed during the build")]
         public abstract void ConflictDirectoryMembershipFingerprint(LoggingContext loggingContext, string path);
 
+        [GeneratedEvent(
+            (ushort)LogEventId.ComChunkerFailulre,
+            EventGenerators = EventGenerators.LocalOnly,
+            EventLevel = Level.Verbose,
+            Keywords = (int)Keywords.UserMessage,
+            EventTask = (ushort)Tasks.Storage,
+            Message = "COM chunker failed to initialize. Using ManagedChunker instead. Exception: '{exception}'")]
+        public abstract void ComChunkerFailulre(LoggingContext loggingContext, string exception);
+
+        [GeneratedEvent(
+            (ushort)LogEventId.ChunkerType,
+            EventGenerators = EventGenerators.LocalOnly,
+            EventLevel = Level.Verbose,
+            Keywords = (int)Keywords.UserMessage,
+            EventTask = (ushort)Tasks.Storage,
+            Message = "Using chunker '{chunkerType}'")]
+        public abstract void ChunkerType(LoggingContext loggingContext, string chunkerType);
+
         public class FileCombinerStats
         {
             public long BeginCount;

Public/Src/Utilities/Storage/Tracing/LogEventId.cs

@@ -112,5 +112,8 @@ namespace BuildXL.Storage.Tracing
         InvalidChangeKindsOfInputChange = 8204,
         InvalidInputChange = 8205,
         InputChangeListFileNotFound = 8206,
+
+        ComChunkerFailulre = 8207,
+        ChunkerType = 8208,
     }
 }

cg/nuget/cgmanifest.json

@@ -33,7 +33,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "ArtifactServices.App.Shared",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -42,7 +42,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "ArtifactServices.App.Shared.Cache",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -195,7 +195,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Drop.App.Core",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -204,7 +204,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Drop.Client",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -402,7 +402,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "ItemStore.Shared",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -429,7 +429,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.ApplicationInsights",
-          "Version": "2.3.0"
+          "Version": "2.11.0"
         }
       }
     },
@@ -1126,6 +1126,15 @@
         }
       }
     },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "Microsoft.Azure.Storage.Common",
+          "Version": "11.1.0"
+        }
+      }
+    },
     {
       "Component": {
         "Type": "NuGet",
@@ -1752,7 +1761,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.IdentityModel.Clients.ActiveDirectory",
-          "Version": "4.5.1"
+          "Version": "5.2.6"
         }
       }
     },
@@ -2211,7 +2220,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.VisualStudio.Services.ArtifactServices.Shared",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -2220,7 +2229,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.VisualStudio.Services.BlobStore.Client",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -2229,7 +2238,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.VisualStudio.Services.BlobStore.Client.Cache",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -2238,7 +2247,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.VisualStudio.Services.Client",
-          "Version": "16.163.0-internal202001081"
+          "Version": "16.169.0-internal202004242"
         }
       }
     },
@@ -2247,7 +2256,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.VisualStudio.Services.InteractiveClient",
-          "Version": "16.163.0-internal202001081"
+          "Version": "16.169.0-internal202004242"
         }
       }
     },
@@ -2589,16 +2598,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Newtonsoft.Json",
-          "Version": "10.0.3"
-        }
-      }
-    },
-    {
-      "Component": {
-        "Type": "NuGet",
-        "NuGet": {
-          "Name": "Newtonsoft.Json",
-          "Version": "11.0.2"
+          "Version": "12.0.3"
         }
       }
     },
@@ -2760,7 +2760,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Symbol.App.Core",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -2769,7 +2769,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Symbol.Client",
-          "Version": "18.163.29708-buildid11260482"
+          "Version": "18.169.30024-buildid11990070"
         }
       }
     },
@@ -3484,6 +3484,15 @@
         }
       }
     },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "System.Private.Uri",
+          "Version": "4.3.2"
+        }
+      }
+    },
     {
       "Component": {
         "Type": "NuGet",

config.dsc

@@ -135,8 +135,7 @@ config({
                 { id: "Microsoft.Tpl.Dataflow", version: "4.5.24" },
                 { id: "Microsoft.TypeScript.Compiler", version: "1.8" },
                 { id: "Microsoft.WindowsAzure.ConfigurationManager", version: "1.8.0.0" },
-                { id: "Newtonsoft.Json", version: "11.0.2" },
-                { id: "Newtonsoft.Json", version: "10.0.3", alias: "Newtonsoft.Json.v10" },
+                { id: "Newtonsoft.Json", version: "12.0.3" },
                 { id: "Newtonsoft.Json.Bson", version: "1.0.1" },
                 { id: "System.Data.SQLite.Core", version: "1.0.109.2" },
                 { id: "System.Reflection.Metadata", version: "1.6.0" },
@@ -204,8 +203,9 @@ config({
                 { id: "Microsoft.NET.Test.Sdk", version: "15.9.0" },
                 { id: "Microsoft.CodeCoverage", version: "15.9.0" },
 
-                { id: "Microsoft.IdentityModel.Clients.ActiveDirectory", version: "4.5.1",
+                { id: "Microsoft.IdentityModel.Clients.ActiveDirectory", version: "5.2.6",
                     dependentPackageIdsToSkip: ["Xamarin.Android.Support.CustomTabs", "Xamarin.Android.Support.v7.AppCompat"] },
+                { id: "System.Private.Uri", version: "4.3.2" },
 
                 // CloudStore dependencies
                 { id: "DeduplicationSigned", version: "1.0.14" },
@@ -235,7 +235,7 @@ config({
 
                 { id: "DotNet.Glob", version: "2.0.3" },
                 { id: "Minimatch", version: "1.1.0.0" },
-                { id: "Microsoft.ApplicationInsights", version: "2.3.0" },
+                { id: "Microsoft.ApplicationInsights", version: "2.11.0", dependentPackageIdsToIgnore: ["System.RunTime.InteropServices"] },
                 { id: "Microsoft.ApplicationInsights.Agent.Intercept", version: "2.0.7" },
                 { id: "Microsoft.ApplicationInsights.DependencyCollector", version: "2.3.0" },
                 { id: "Microsoft.ApplicationInsights.PerfCounterCollector", version: "2.3.0" },

config.microsoftInternal.dsc

@@ -23,23 +23,24 @@ export const pkgs = isMicrosoftInternal ? [
     { id: "WindowsSdk.Corext", version: "10.0.16299.1", alias: "Windows.Sdk", osSkip: [ "macOS", "unix" ] },
 
     // Artifact packages and dependencies
-    { id: "Microsoft.VisualStudio.Services.ArtifactServices.Shared", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
-    { id: "Microsoft.VisualStudio.Services.BlobStore.Client", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces"] },
-    { id: "Microsoft.VisualStudio.Services.Client", version: "16.163.0-internal202001081", dependentPackageIdsToSkip: [ "Microsoft.Net.Http", "Microsoft.AspNet.WebApi.Client", "System.Security.Cryptography.OpenSsl", "System.Security.Principal.Windows" ] },
-    { id: "Microsoft.VisualStudio.Services.InteractiveClient", version: "16.163.0-internal202001081", dependentPackageIdsToSkip: [ "Ben.Demystifier" ], dependentPackageIdsToIgnore: [ "Ben.Demystifier" ] },
+    { id: "Microsoft.VisualStudio.Services.ArtifactServices.Shared", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "Microsoft.Azure.Cosmos.Table", "Microsoft.Azure.Storage.Blob"] },
+    { id: "Microsoft.VisualStudio.Services.BlobStore.Client", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces"] },
+    { id: "Microsoft.VisualStudio.Services.Client", version: "16.169.0-internal202004242", dependentPackageIdsToSkip: [ "Microsoft.Net.Http", "Microsoft.AspNet.WebApi.Client", "System.Security.Cryptography.OpenSsl", "System.Security.Principal.Windows" ] },
+    { id: "Microsoft.VisualStudio.Services.InteractiveClient", version: "16.169.0-internal202004242", dependentPackageIdsToSkip: [ "Ben.Demystifier" ], dependentPackageIdsToIgnore: [ "Ben.Demystifier" ] },
+    { id: "Microsoft.Azure.Storage.Common", version:"11.1.0" },
 
     // DropDaemon Artifact dependencies.
     // Here, even though the packages depend on Cache bits other than Hashing, we make sure that the codepaths that actually depend on them are never activated. This is to ensure that there is no cyclic dependency between BXL and AzureDevOps.
     // This is further enforced by not including Cache bits in DropDaemon, other than BuildXL.Cache.Hashing.
-    { id: "ArtifactServices.App.Shared", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
-    { id: "ArtifactServices.App.Shared.Cache", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
-    { id: "Drop.App.Core", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
-    { id: "Drop.Client", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
-    { id: "ItemStore.Shared", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
-    { id: "Microsoft.VisualStudio.Services.BlobStore.Client.Cache", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
+    { id: "ArtifactServices.App.Shared", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
+    { id: "ArtifactServices.App.Shared.Cache", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
+    { id: "Drop.App.Core", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
+    { id: "Drop.Client", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
+    { id: "ItemStore.Shared", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
+    { id: "Microsoft.VisualStudio.Services.BlobStore.Client.Cache", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
     { id: "Microsoft.Windows.Debuggers.SymstoreInterop", version: "1.0.1" },
-    { id: "Symbol.App.Core", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
-    { id: "Symbol.Client", version: "18.163.29708-buildid11260482", dependentPackageIdsToSkip: ["*"] },
+    { id: "Symbol.App.Core", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.library.forAzDev"] },
+    { id: "Symbol.Client", version: "18.169.30024-buildid11990070", dependentPackageIdsToSkip: ["*"] },
 
     // CloudTest internal dependencies
     { id: "GvfsTestHelpersForBuildXL", version: "0.1.0"},