diff --git a/Public/Src/Cache/ContentStore/Vsts/BlobContentSession.cs b/Public/Src/Cache/ContentStore/Vsts/BlobContentSession.cs index 9b03f6d4c..b81be8525 100644 --- a/Public/Src/Cache/ContentStore/Vsts/BlobContentSession.cs +++ b/Public/Src/Cache/ContentStore/Vsts/BlobContentSession.cs @@ -301,7 +301,7 @@ namespace BuildXL.Cache.ContentStore.Vsts { var endDateTime = DateTime.UtcNow + TimeToKeepContent; await BlobStoreHttpClient.UploadAndReferenceBlobWithRetriesAsync( - ToVstsBlobIdentifier(BuildXL.Cache.ContentStore.Hashing.BlobIdentifierHelperExtensions.ToBlobIdentifier(contentHash)), + contentHash.ToBlobIdentifier(), stream, new BlobReference(endDateTime), context, diff --git a/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs b/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs index d2dd9365c..7e04754f7 100644 --- a/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs +++ b/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs @@ -32,10 +32,8 @@ using Microsoft.VisualStudio.Services.Content.Common; using Microsoft.VisualStudio.Services.WebApi; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; -using BlobIdentifier = BuildXL.Cache.ContentStore.Hashing.BlobIdentifier; using ByteArrayPool = Microsoft.VisualStudio.Services.BlobStore.Common.ByteArrayPool; using OperationContext = BuildXL.Cache.ContentStore.Tracing.Internal.OperationContext; -using VstsBlobIdentifier = Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifier; #nullable enable @@ -313,15 +311,10 @@ namespace BuildXL.Cache.ContentStore.Vsts protected override Task>>> PlaceFileCoreAsync(OperationContext context, IReadOnlyList hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) => throw new NotImplementedException(); - /// - /// Converts a ContentStore blob id to an artifact BlobId - /// - protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier) => new VstsBlobIdentifier(blobIdentifier.Bytes); - private async Task>>> UpdateBlobStoreAsync(OperationContext context, IReadOnlyList contentHashes, DateTime endDateTime) { // Convert missing content hashes to blob Ids - var blobIds = contentHashes.Select(c => ToVstsBlobIdentifier(BuildXL.Cache.ContentStore.Hashing.BlobIdentifierHelperExtensions.ToBlobIdentifier(c))).ToList(); + var blobIds = contentHashes.Select(contentHash => contentHash.ToBlobIdentifier()).ToList(); // Call TryReference on the blob ids var references = blobIds.Distinct().ToDictionary( @@ -539,18 +532,18 @@ namespace BuildXL.Cache.ContentStore.Vsts if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var uri)) { _blobCounters[Counters.VstsDownloadUriFetchedFromRemote].Increment(); - var blobId = BuildXL.Cache.ContentStore.Hashing.BlobIdentifierHelperExtensions.ToBlobIdentifier(contentHash); + var blobId = contentHash.ToBlobIdentifier(); var mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternal", innerCts => BlobStoreHttpClient.GetDownloadUrisAsync( - new[] { ToVstsBlobIdentifier(blobId) }, + new[] { blobId }, EdgeCache.NotAllowed, cancellationToken: innerCts), context.Token).ConfigureAwait(false); - if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri)) + if (mappings == null || !mappings.TryGetValue(blobId, out uri)) { return null; } diff --git a/Public/Src/Cache/ContentStore/Vsts/BlobStoreHttpClientExtensions.cs b/Public/Src/Cache/ContentStore/Vsts/BlobStoreHttpClientExtensions.cs index 8bc793162..534d77917 100644 --- a/Public/Src/Cache/ContentStore/Vsts/BlobStoreHttpClientExtensions.cs +++ b/Public/Src/Cache/ContentStore/Vsts/BlobStoreHttpClientExtensions.cs @@ -11,10 +11,13 @@ using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Cache.ContentStore.Interfaces.Tracing; using Microsoft.VisualStudio.Services.BlobStore.Common; using Microsoft.VisualStudio.Services.BlobStore.WebApi; using Microsoft.VisualStudio.Services.Content.Common; +using VsoHash = Microsoft.VisualStudio.Services.BlobStore.Common.VsoHash; +using BlobIdentifierWithBlocks = Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifierWithBlocks; namespace BuildXL.Cache.ContentStore.Vsts { diff --git a/Public/Src/Cache/ContentStore/Vsts/DedupContentSession.cs b/Public/Src/Cache/ContentStore/Vsts/DedupContentSession.cs index b8f107e86..bb00822db 100644 --- a/Public/Src/Cache/ContentStore/Vsts/DedupContentSession.cs +++ b/Public/Src/Cache/ContentStore/Vsts/DedupContentSession.cs @@ -17,7 +17,6 @@ using BuildXL.Cache.ContentStore.Utils; using BuildXL.Utilities.Tracing; using Microsoft.VisualStudio.Services.BlobStore.Common; using Microsoft.VisualStudio.Services.BlobStore.WebApi; -using VstsDedupIdentifier = Microsoft.VisualStudio.Services.BlobStore.Common.DedupIdentifier; using VstsFileSystem = Microsoft.VisualStudio.Services.Content.Common.FileSystem; namespace BuildXL.Cache.ContentStore.Vsts @@ -253,7 +252,7 @@ namespace BuildXL.Cache.ContentStore.Vsts private Task PutNodeAsync(OperationContext context, DedupNode dedupNode, AbsolutePath path) { - var dedupIdentifier = dedupNode.GetDedupId(); + var dedupIdentifier = dedupNode.GetDedupIdentifier(HashType.Dedup64K); return TryGatedArtifactOperationAsync( context, @@ -261,14 +260,14 @@ namespace BuildXL.Cache.ContentStore.Vsts "DedupUploadSession.UploadAsync", async innerCts => { - await _uploadSession.UploadAsync(dedupNode, new Dictionary { { dedupIdentifier, path.Path } }, innerCts); + await _uploadSession.UploadAsync(dedupNode, new Dictionary { { dedupIdentifier, path.Path } }, innerCts); return null; }); } private Task PutChunkAsync(OperationContext context, DedupNode dedupNode, AbsolutePath path) { - var dedupIdentifier = dedupNode.GetDedupId(); + var dedupIdentifier = dedupNode.GetDedupIdentifier(HashType.Dedup64K); return TryGatedArtifactOperationAsync( context, dedupIdentifier.ValueString, diff --git a/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs b/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs index 0015e2bad..e1295e74a 100644 --- a/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs +++ b/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs @@ -28,11 +28,8 @@ using BuildXL.Utilities.Tracing; using Microsoft.VisualStudio.Services.BlobStore.Common; using Microsoft.VisualStudio.Services.BlobStore.WebApi; using Microsoft.WindowsAzure.Storage; -using BlobIdentifier = BuildXL.Cache.ContentStore.Hashing.BlobIdentifier; using FileInfo = System.IO.FileInfo; using OperationContext = BuildXL.Cache.ContentStore.Tracing.Internal.OperationContext; -using VstsDedupIdentifier = Microsoft.VisualStudio.Services.BlobStore.Common.DedupIdentifier; -using VstsBlobIdentifier = Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifier; namespace BuildXL.Cache.ContentStore.Vsts { @@ -184,8 +181,8 @@ namespace BuildXL.Cache.ContentStore.Vsts return pinResult; } - VstsBlobIdentifier blobId = ToVstsBlobIdentifier(BuildXL.Cache.ContentStore.Hashing.BlobIdentifierHelperExtensions.ToBlobIdentifier(contentHash)); - VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier(); + BlobIdentifier blobId = contentHash.ToBlobIdentifier(); + DedupIdentifier dedupId = blobId.ToDedupIdentifier(); if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { @@ -244,7 +241,7 @@ namespace BuildXL.Cache.ContentStore.Vsts try { PinResult pinResult; - var dedupId = ToVstsBlobIdentifier(BuildXL.Cache.ContentStore.Hashing.BlobIdentifierHelperExtensions.ToBlobIdentifier(contentHash)).ToDedupIdentifier(); + var dedupId = contentHash.ToBlobIdentifier().ToDedupIdentifier(); if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { pinResult = await TryPinChunkAsync(context, dedupId, keepUntil); @@ -419,11 +416,6 @@ namespace BuildXL.Cache.ContentStore.Vsts throw new NotImplementedException(); } - /// - /// Converts a ContentStore BlobId to an Artifact BlobId - /// - protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier) => new VstsBlobIdentifier(blobIdentifier.Bytes); - private Task PlaceFileInternalAsync( OperationContext context, ContentHash contentHash, string path, FileMode fileMode) { @@ -460,8 +452,8 @@ namespace BuildXL.Cache.ContentStore.Vsts private async Task GetFileWithDedupAsync(OperationContext context, ContentHash contentHash, string path) { - VstsBlobIdentifier blobId = ToVstsBlobIdentifier(BuildXL.Cache.ContentStore.Hashing.BlobIdentifierHelperExtensions.ToBlobIdentifier(contentHash)); - VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier(); + BlobIdentifier blobId = contentHash.ToBlobIdentifier(); + DedupIdentifier dedupId = blobId.ToDedupIdentifier(); try { @@ -543,7 +535,7 @@ namespace BuildXL.Cache.ContentStore.Vsts /// /// Updates expiry of single chunk in DedupStore if it exists. /// - private async Task TryPinChunkAsync(OperationContext context, VstsDedupIdentifier dedupId, DateTime keepUntil) + private async Task TryPinChunkAsync(OperationContext context, DedupIdentifier dedupId, DateTime keepUntil) { try { @@ -572,7 +564,7 @@ namespace BuildXL.Cache.ContentStore.Vsts /// 2) All children exist and have sufficient TTL /// If children have insufficient TTL, attempt to extend the expiry of all children before pinning. /// - private async Task TryPinNodeAsync(OperationContext context, VstsDedupIdentifier dedupId, DateTime keepUntil) + private async Task TryPinNodeAsync(OperationContext context, DedupIdentifier dedupId, DateTime keepUntil) { TryReferenceNodeResponse referenceResult; try @@ -615,10 +607,10 @@ namespace BuildXL.Cache.ContentStore.Vsts /// /// Attempt to update expiry of all children. Pin parent node if all children were extended successfully. /// - private async Task TryPinChildrenAsync(OperationContext context, VstsDedupIdentifier parentNode, IEnumerable dedupIdentifiers, DateTime keepUntil) + private async Task TryPinChildrenAsync(OperationContext context, DedupIdentifier parentNode, IEnumerable dedupIdentifiers, DateTime keepUntil) { - var chunks = new List(); - var nodes = new List(); + var chunks = new List(); + var nodes = new List(); foreach (var id in dedupIdentifiers) { @@ -652,7 +644,7 @@ namespace BuildXL.Cache.ContentStore.Vsts /// Recursively attempt to update expiry of all nodes and their children. /// Returns success only if all children of each node are found and extended. /// - private async Task TryPinNodesAsync(OperationContext context, IEnumerable dedupIdentifiers, DateTime keepUntil) + private async Task TryPinNodesAsync(OperationContext context, IEnumerable dedupIdentifiers, DateTime keepUntil) { if (!dedupIdentifiers.Any()) { @@ -660,7 +652,7 @@ namespace BuildXL.Cache.ContentStore.Vsts } // TODO: Support batched TryKeepUntilReferenceNodeAsync in Artifact. (bug 1428612) - var tryReferenceBlock = new TransformBlock( + var tryReferenceBlock = new TransformBlock( async dedupId => await TryPinNodeAsync(context, dedupId, keepUntil), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DefaultMaxParallelism }); @@ -682,7 +674,7 @@ namespace BuildXL.Cache.ContentStore.Vsts /// /// Update all chunks if they exist. Returns success only if all chunks are found and extended. /// - private async Task TryPinChunksAsync(OperationContext context, IEnumerable dedupIdentifiers, DateTime keepUntil) + private async Task TryPinChunksAsync(OperationContext context, IEnumerable dedupIdentifiers, DateTime keepUntil) { if (!dedupIdentifiers.Any()) { @@ -690,7 +682,7 @@ namespace BuildXL.Cache.ContentStore.Vsts } // TODO: Support batched TryKeepUntilReferenceChunkAsync in Artifact. (bug 1428612) - var tryReferenceBlock = new TransformBlock( + var tryReferenceBlock = new TransformBlock( async dedupId => await TryPinChunkAsync(context, dedupId, keepUntil), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DefaultMaxParallelism }); @@ -712,7 +704,7 @@ namespace BuildXL.Cache.ContentStore.Vsts /// /// Checks the current keepUntil of a node. Returns null if the node is not found. /// - protected async Task> CheckNodeKeepUntilAsync(OperationContext context, VstsDedupIdentifier dedupId) + protected async Task> CheckNodeKeepUntilAsync(OperationContext context, DedupIdentifier dedupId) { TryReferenceNodeResponse referenceResult; try diff --git a/Public/Src/Cache/MemoizationStore/Vsts/Exceptions.cs b/Public/Src/Cache/MemoizationStore/Vsts/Exceptions.cs index e35e62dc6..9ea1aa869 100644 --- a/Public/Src/Cache/MemoizationStore/Vsts/Exceptions.cs +++ b/Public/Src/Cache/MemoizationStore/Vsts/Exceptions.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Runtime.Serialization; +using BuildXL.Cache.ContentStore.Hashing; using Microsoft.VisualStudio.Services.BlobStore.Common; using Microsoft.VisualStudio.Services.Common; diff --git a/Public/Src/Tools/DropDaemon/DropFile.cs b/Public/Src/Tools/DropDaemon/DropFile.cs index adc5dd5e2..5638cbd32 100644 --- a/Public/Src/Tools/DropDaemon/DropFile.cs +++ b/Public/Src/Tools/DropDaemon/DropFile.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System.Diagnostics.ContractsLight; +using BuildXL.Cache.ContentStore.Hashing; using Microsoft.VisualStudio.Services.BlobStore.Common; namespace Tool.DropDaemon diff --git a/Public/Src/Tools/DropDaemon/DropItemForFile.cs b/Public/Src/Tools/DropDaemon/DropItemForFile.cs index ae9748bc6..a5186cbaa 100644 --- a/Public/Src/Tools/DropDaemon/DropItemForFile.cs +++ b/Public/Src/Tools/DropDaemon/DropItemForFile.cs @@ -6,6 +6,7 @@ using System.Diagnostics.ContractsLight; using System.IO; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Storage; using Microsoft.VisualStudio.Services.BlobStore.Common; using Tool.ServicePipDaemon; @@ -145,7 +146,7 @@ namespace Tool.DropDaemon case BuildXL.Cache.ContentStore.Hashing.HashType.DedupSingleChunk: return new ChunkDedupIdentifier(contentHash.ToHashByteArray()).ToBlobIdentifier(); case BuildXL.Cache.ContentStore.Hashing.HashType.DedupNode: - return new NodeDedupIdentifier(contentHash.ToHashByteArray()).ToBlobIdentifier(); + return new NodeDedupIdentifier(contentHash.ToHashByteArray(), NodeAlgorithmId.Node64K).ToBlobIdentifier(); default: throw new ArgumentException($"ContentHash has unsupported type when converting to BlobIdentifier: {contentHash.HashType}"); } diff --git a/Public/Src/Tools/DropDaemon/IDropItem.cs b/Public/Src/Tools/DropDaemon/IDropItem.cs index 2178349c5..cf443eaa6 100644 --- a/Public/Src/Tools/DropDaemon/IDropItem.cs +++ b/Public/Src/Tools/DropDaemon/IDropItem.cs @@ -4,8 +4,8 @@ using System.Diagnostics.Contracts; using System.IO; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Hashing; using JetBrains.Annotations; -using Microsoft.VisualStudio.Services.BlobStore.Common; using Pure = System.Diagnostics.Contracts.PureAttribute; namespace Tool.DropDaemon diff --git a/Public/Src/Tools/DropDaemon/ReloadingDropServiceClient.cs b/Public/Src/Tools/DropDaemon/ReloadingDropServiceClient.cs index f7a09e4df..7f8797d5f 100644 --- a/Public/Src/Tools/DropDaemon/ReloadingDropServiceClient.cs +++ b/Public/Src/Tools/DropDaemon/ReloadingDropServiceClient.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Ipc.Interfaces; using Microsoft.VisualStudio.Services.BlobStore.Common; using Microsoft.VisualStudio.Services.BlobStore.WebApi; diff --git a/Public/Src/Tools/DropDaemon/VsoClient.cs b/Public/Src/Tools/DropDaemon/VsoClient.cs index f703882b3..62ecdbd00 100644 --- a/Public/Src/Tools/DropDaemon/VsoClient.cs +++ b/Public/Src/Tools/DropDaemon/VsoClient.cs @@ -9,6 +9,7 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Ipc.Common; using BuildXL.Ipc.ExternalApi; using BuildXL.Ipc.Interfaces; diff --git a/Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/FileConsumptionAnalyzer.cs b/Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/FileConsumptionAnalyzer.cs index fba9a45a0..88f1403ba 100644 --- a/Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/FileConsumptionAnalyzer.cs +++ b/Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/FileConsumptionAnalyzer.cs @@ -461,7 +461,7 @@ namespace BuildXL.Execution.Analyzer } } - m_sizeByExtension.AddOrUpdate(entry.Key.GetExtension(m_analyzer.PathTable), size, (_, oldValue) => oldValue + size); + m_sizeByExtension.AddOrUpdate(entry.Key.GetExtension(m_analyzer.PathTable), size, (long _, long oldValue) => oldValue + size); } } diff --git a/Public/Src/Tools/SymbolDaemon/ReloadingSymbolClient.cs b/Public/Src/Tools/SymbolDaemon/ReloadingSymbolClient.cs index 4b54fc592..260bc74fe 100644 --- a/Public/Src/Tools/SymbolDaemon/ReloadingSymbolClient.cs +++ b/Public/Src/Tools/SymbolDaemon/ReloadingSymbolClient.cs @@ -6,10 +6,12 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BuildXL.Ipc.Interfaces; +using BuildXL.Cache.ContentStore.Hashing; using Microsoft.VisualStudio.Services.BlobStore.Common; using Microsoft.VisualStudio.Services.Content.Common; using Microsoft.VisualStudio.Services.Symbol.App.Core; using Microsoft.VisualStudio.Services.Symbol.WebApi; +using BlobIdentifierWithBlocks = Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifierWithBlocks; namespace Tool.ServicePipDaemon { @@ -94,12 +96,12 @@ namespace Tool.ServicePipDaemon } /// - public BlobIdentifier GetBlobIdentifier(string filename) + public BlobIdentifier GetBlobIdentifier(string filename, bool useChunkDedup) { var instance = GetCurrentVersionedValue(); // not retrying this since it does not perform any calls over the network - return instance.Value.GetBlobIdentifier(filename); + return instance.Value.GetBlobIdentifier(filename, useChunkDedup); } /// @@ -180,7 +182,7 @@ namespace Tool.ServicePipDaemon } /// - public Task UploadFileAsync(IDomainId domainId, Uri blobStoreUri, string requestId, string filename, BlobIdentifier blobIdentifier, CancellationToken cancellationToken) + public Task UploadFileAsync(IDomainId domainId, Uri blobStoreUri, string requestId, string filename, BlobIdentifier blobIdentifier, CancellationToken cancellationToken) { return RetryAsync( nameof(ISymbolServiceClient.UploadFileAsync), diff --git a/Public/Src/Tools/SymbolDaemon/SymbolDaemon.cs b/Public/Src/Tools/SymbolDaemon/SymbolDaemon.cs index 0bc4416ac..8afa6f4da 100644 --- a/Public/Src/Tools/SymbolDaemon/SymbolDaemon.cs +++ b/Public/Src/Tools/SymbolDaemon/SymbolDaemon.cs @@ -434,7 +434,7 @@ namespace Tool.SymbolDaemon { ContentHash.TryParse(reader.ReadLine(), out var hash); Contract.Assert(hash.HashType == HashType.Vso0); - var blobIdentifier = new Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifier(hash.ToHashByteArray()); + var blobIdentifier = new BlobIdentifier(hash.ToHashByteArray()); int debugEntryCount = int.Parse(reader.ReadLine()); var symbols = new HashSet(debugEntryCount, DebugEntryDataComparer.Instance); @@ -480,7 +480,7 @@ namespace Tool.SymbolDaemon var result = new DebugEntryData() { - BlobIdentifier = blocks[0].Length == 0 ? null : Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifier.Deserialize(blocks[0]), + BlobIdentifier = blocks[0].Length == 0 ? null : BlobIdentifier.Deserialize(blocks[0]), ClientKey = string.IsNullOrEmpty(blocks[1]) ? null : blocks[1], InformationLevel = (DebugInformationLevel)int.Parse(blocks[2]) }; diff --git a/Public/Src/Tools/SymbolDaemon/SymbolFile.cs b/Public/Src/Tools/SymbolDaemon/SymbolFile.cs index 0a33b9c79..7cc71f2a2 100644 --- a/Public/Src/Tools/SymbolDaemon/SymbolFile.cs +++ b/Public/Src/Tools/SymbolDaemon/SymbolFile.cs @@ -90,7 +90,7 @@ namespace Tool.SymbolDaemon if (debugEntries != null) { - var blobIdentifier = new Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifier(hash.ToHashByteArray()); + var blobIdentifier = new BlobIdentifier(hash.ToHashByteArray()); Contract.Assert(debugEntries.All(e => e.BlobIdentifier == blobIdentifier)); m_debugEntries = new List(debugEntries); } @@ -108,7 +108,7 @@ namespace Tool.SymbolDaemon Contract.Requires(entries != null); // check that either all entries are missing the blobId, or all the entries have the same blobId and that blobId matches this file - var blobIdentifier = new Microsoft.VisualStudio.Services.BlobStore.Common.BlobIdentifier(Hash.ToHashByteArray()); + var blobIdentifier = new BlobIdentifier(Hash.ToHashByteArray()); Contract.Assert(entries.All(e => e.BlobIdentifier == null) || entries.All(e => e.BlobIdentifier == blobIdentifier)); // ensure that BlobIdentifier is not null diff --git a/Public/Src/Tools/SymbolDaemon/SymbolIndexer.cs b/Public/Src/Tools/SymbolDaemon/SymbolIndexer.cs index 7740872e5..d7437edd1 100644 --- a/Public/Src/Tools/SymbolDaemon/SymbolIndexer.cs +++ b/Public/Src/Tools/SymbolDaemon/SymbolIndexer.cs @@ -30,7 +30,9 @@ namespace Tool.SymbolDaemon var entries = m_symstoreUtil.GetDebugEntryData( file.FullName, new[] { file.FullName }, - calculateBlobId); + calculateBlobId, + // Currently, only file-deduped (VsoHash) symbols are supported + isChunked: false); return entries; } diff --git a/Public/Src/Tools/SymbolDaemon/VsoSymbolClient.cs b/Public/Src/Tools/SymbolDaemon/VsoSymbolClient.cs index 84b770541..8a6fe094d 100644 --- a/Public/Src/Tools/SymbolDaemon/VsoSymbolClient.cs +++ b/Public/Src/Tools/SymbolDaemon/VsoSymbolClient.cs @@ -7,6 +7,7 @@ using System.Diagnostics.ContractsLight; using System.Linq; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Ipc.Common; using BuildXL.Ipc.ExternalApi; using BuildXL.Ipc.Interfaces; @@ -366,12 +367,12 @@ namespace Tool.SymbolDaemon batchedFile.File.FullFilePath, entry.BlobIdentifier, CancellationToken); - batchedFile.SetBlobDetails(uploadResult); + batchedFile.SetBlobIdentifier(uploadResult); } }); // need to update entries before calling associate the second time - entriesWithMissingBlobs.ForEach(entry => entry.BlobDetails = missingBlobsToFilesMap[entry.BlobIdentifier].BlobDetails); + entriesWithMissingBlobs.ForEach(entry => missingBlobsToFilesMap[entry.BlobIdentifier].BlobIdentifier.UpdateDebugEntryBlobReference(entry)); using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateAfterUploadTime)) { @@ -481,7 +482,7 @@ namespace Tool.SymbolDaemon public SymbolFile File { get; } public TaskSourceSlim ResultTaskSource { get; } - public BlobIdentifierWithBlocks BlobDetails { get; private set; } + public SymbolBlobIdentifier BlobIdentifier { get; private set; } public BatchedSymbolFile(SymbolFile file) { @@ -489,11 +490,11 @@ namespace Tool.SymbolDaemon ResultTaskSource = TaskSourceSlim.Create(); } - public void SetBlobDetails(BlobIdentifierWithBlocks details) + public void SetBlobIdentifier(SymbolBlobIdentifier blobIdentifier) { lock (m_lock) { - BlobDetails = details; + BlobIdentifier = blobIdentifier; } } } diff --git a/Public/Src/Tools/UnitTests/DropDaemon/ReloadingDropServiceClientTest.cs b/Public/Src/Tools/UnitTests/DropDaemon/ReloadingDropServiceClientTest.cs index 455c993ae..35c9f3e25 100644 --- a/Public/Src/Tools/UnitTests/DropDaemon/ReloadingDropServiceClientTest.cs +++ b/Public/Src/Tools/UnitTests/DropDaemon/ReloadingDropServiceClientTest.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Ipc.Common; using BuildXL.Ipc.Interfaces; using Microsoft.VisualStudio.Services.BlobStore.Common; diff --git a/cg/nuget/cgmanifest.json b/cg/nuget/cgmanifest.json index 9d46a5301..f53b76285 100644 --- a/cg/nuget/cgmanifest.json +++ b/cg/nuget/cgmanifest.json @@ -33,7 +33,7 @@ "Type": "NuGet", "NuGet": { "Name": "ArtifactServices.App.Shared", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -42,7 +42,7 @@ "Type": "NuGet", "NuGet": { "Name": "ArtifactServices.App.Shared.Cache", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -213,7 +213,7 @@ "Type": "NuGet", "NuGet": { "Name": "Drop.App.Core", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -222,7 +222,7 @@ "Type": "NuGet", "NuGet": { "Name": "Drop.Client", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -384,7 +384,7 @@ "Type": "NuGet", "NuGet": { "Name": "ItemStore.Shared", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -1923,7 +1923,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.VisualStudio.Services.ArtifactServices.Shared", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -1932,7 +1932,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.VisualStudio.Services.BlobStore.Client", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -1941,7 +1941,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.VisualStudio.Services.BlobStore.Client.Cache", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -1950,7 +1950,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.VisualStudio.Services.Client", - "Version": "16.176.0-internal202009221" + "Version": "16.183.0-internal202102091" } } }, @@ -1959,7 +1959,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.VisualStudio.Services.InteractiveClient", - "Version": "16.176.0-internal202009221" + "Version": "16.183.0-internal202102091" } } }, @@ -2490,7 +2490,7 @@ "Type": "NuGet", "NuGet": { "Name": "Symbol.App.Core", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, @@ -2499,7 +2499,7 @@ "Type": "NuGet", "NuGet": { "Name": "Symbol.Client", - "Version": "18.176.30522-buildid13159358" + "Version": "18.183.31009-buildid14325558" } } }, diff --git a/config.microsoftInternal.dsc b/config.microsoftInternal.dsc index 4f730b9ff..a81bfd9d4 100644 --- a/config.microsoftInternal.dsc +++ b/config.microsoftInternal.dsc @@ -3,8 +3,8 @@ const isMicrosoftInternal = Environment.getFlag("[Sdk.BuildXL]microsoftInternal"); -const artifactNugetVersion = "18.176.30522-buildid13159358"; -const azureDevopsNugetVersion = "16.176.0-internal202009221"; +const artifactNugetVersion = "18.183.31009-buildid14325558"; +const azureDevopsNugetVersion = "16.183.0-internal202102091"; // These packages are Microsoft internal packages. // These consist of internally repackaged products that we can't push to a public feed and have to rely on users installing locally. @@ -45,7 +45,7 @@ export const pkgs = isMicrosoftInternal ? [ { id: "Microsoft.VisualStudio.Services.BlobStore.Client.Cache", version: artifactNugetVersion, dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.Utilities"] }, { id: "Microsoft.Windows.Debuggers.SymstoreInterop", version: "1.0.1" }, { id: "Symbol.App.Core", version: artifactNugetVersion, dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing", "BuildXL.Cache.Interfaces", "BuildXL.Cache.Libraries", "BuildXL.Utilities"] }, - { id: "Symbol.Client", version: artifactNugetVersion, dependentPackageIdsToSkip: ["*"] }, + { id: "Symbol.Client", version: artifactNugetVersion, dependentPackageIdsToSkip: ["*"], dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] }, { id: "TransientFaultHandling.Core", version: "5.1.1209.1" }, // CloudTest internal dependencies